1use std::collections::{BTreeMap, HashMap, VecDeque};
2use std::path::{Path, PathBuf};
3use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
4use std::sync::{Arc, Mutex as StdMutex};
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use async_trait::async_trait;
8use base64::Engine;
9use futures_util::StreamExt;
10use reqwest::header::{
11 ACCEPT_ENCODING, AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, LOCATION,
12};
13use reqwest::redirect::Policy;
14use reqwest::{Client, Response, StatusCode};
15use shaul_core::{
16 BackendCapabilities, Checksum, ChecksumAlgorithm, Config, DownloadBackend, DownloadOutcome,
17 DownloadRequest, Error, EventKind, EventSink, FileOutcome, FileResume, Header, Resource,
18 ResourceFile, ResourceKind, Result, ResumeCheckpoint, TaskId, TaskResume, Validator,
19 VerifyMode, completed_bytes, merge_ranges, missing_ranges, part_path, sanitize_file_name,
20 split_range,
21};
22use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
23use tokio::sync::Mutex;
24use tokio_util::sync::CancellationToken;
25use url::Url;
26
27use crate::budget::{ConnectionBudget, ConnectionPermit};
28use crate::filename::parse_content_disposition;
29use crate::rate::{RateLimiter, RateLimiters};
30
31const MAX_REDIRECTS: usize = 10;
32
33const CAPTURE_HEADERS: &[&str] = &[
34 "content-disposition",
35 "etag",
36 "last-modified",
37 "content-length",
38 "accept-ranges",
39 "content-range",
40 "digest",
41 "content-md5",
42 "x-linked-etag",
43 "x-linked-size",
44 "x-repo-commit",
45 "x-xet-hash",
46 "x-xet-token-expiration",
47 "x-xet-refresh-route",
48 "x-error-code",
49];
50
51const RESERVED_HEADERS: &[&str] = &[
52 "range",
53 "if-range",
54 "host",
55 "content-length",
56 "accept-encoding",
57 "connection",
58 "transfer-encoding",
59];
60
61type TaskRates = Arc<StdMutex<HashMap<TaskId, (u64, Arc<RateLimiter>)>>>;
62
63#[derive(Clone)]
65pub struct HttpBackend {
66 client: Client,
67 budget: Arc<ConnectionBudget>,
68 global_rate: Option<Arc<RateLimiter>>,
69 task_rates: TaskRates,
70}
71
72impl HttpBackend {
73 pub fn new(config: &Config) -> Result<Self> {
76 let mut builder = Client::builder()
77 .redirect(Policy::none())
78 .connect_timeout(Duration::from_secs(
79 config.network.connect_timeout_secs.max(1),
80 ))
81 .pool_max_idle_per_host(config.download.per_host.max(1))
82 .user_agent(config.network.user_agent.clone());
83 if let Some(proxy) = proxy_from_config(config)? {
84 builder = builder.proxy(proxy);
85 }
86 let client = builder
87 .build()
88 .map_err(|error| Error::Backend(format!("failed to build HTTP client: {error}")))?;
89 Ok(Self {
90 client,
91 budget: Arc::new(ConnectionBudget::new(
92 config.download.global,
93 config.download.per_host_active,
94 )),
95 global_rate: config
96 .download
97 .rate_limit
98 .filter(|value| *value > 0)
99 .map(|rate| Arc::new(RateLimiter::new(rate))),
100 task_rates: Arc::new(StdMutex::new(HashMap::new())),
101 })
102 }
103
104 fn request_rate(&self, request: &DownloadRequest) -> RateLimiters {
105 let task = request
106 .options
107 .task_rate_limit
108 .filter(|value| *value > 0)
109 .map(|rate| {
110 let mut rates = self
111 .task_rates
112 .lock()
113 .unwrap_or_else(|poisoned| poisoned.into_inner());
114 if let Some((existing, limiter)) = rates.get(&request.task_id)
115 && *existing == rate
116 {
117 return limiter.clone();
118 }
119 let limiter = Arc::new(RateLimiter::new(rate));
120 rates.insert(request.task_id, (rate, limiter.clone()));
121 limiter
122 });
123 RateLimiters::new(self.global_rate.clone(), task)
124 }
125
126 pub async fn download_file(
130 &self,
131 request: &DownloadRequest,
132 file: &ResourceFile,
133 task_resume: &mut TaskResume,
134 ) -> Result<FileOutcome> {
135 let checksum = request
136 .options
137 .checksum
138 .clone()
139 .or_else(|| file.checksum.clone());
140 let mut reference: Option<Validator> = None;
141 let mut last_error: Option<Error> = None;
142 let mut urls = vec![file.url.clone()];
143 for mirror in &request.options.mirrors {
144 if !matches!(mirror.scheme(), "http" | "https") {
145 warning(request, format!("ignoring non-HTTP mirror URL: {mirror}"));
146 continue;
147 }
148 if !urls.contains(mirror) {
149 urls.push(mirror.clone());
150 }
151 }
152
153 for (index, url) in urls.iter().enumerate() {
154 let mut candidate = file.clone();
155 candidate.url = url.clone();
156 let probe = match self.probe(request, &candidate).await {
157 Ok(probe) => probe,
158 Err(error) if error.is_cancelled() => return Err(error),
159 Err(error) => {
160 warning(request, format!("download URL {url} failed: {error}"));
161 last_error = Some(error);
162 continue;
163 }
164 };
165 let validator = probe.validator();
166 if let Some(reference) = &reference {
167 if !mirror_compatible(reference, &validator, checksum.is_some()) {
168 warning(
169 request,
170 format!("mirror {url} is incompatible with the primary response"),
171 );
172 last_error = Some(Error::Backend(format!(
173 "mirror {url} is incompatible with the primary response"
174 )));
175 continue;
176 }
177 } else {
178 reference = Some(validator.clone());
179 }
180 if index > 0 {
181 warning(request, format!("falling back to mirror {url}"));
182 }
183 match self
184 .download_file_with_probe(request, &candidate, task_resume, probe)
185 .await
186 {
187 Ok(outcome) => return Ok(outcome),
188 Err(error) if error.is_cancelled() => return Err(error),
189 Err(error) => {
190 warning(request, format!("download URL {url} failed: {error}"));
191 last_error = Some(error);
192 }
193 }
194 }
195 Err(last_error
196 .unwrap_or_else(|| Error::Backend("no HTTP download URL was available".to_string())))
197 }
198
199 #[tracing::instrument(
200 name = "shaul.http.file",
201 level = "debug",
202 skip(self, request, file, task_resume, probe),
203 fields(task_id = %request.task_id, file = %file.path)
204 )]
205 async fn download_file_with_probe(
206 &self,
207 request: &DownloadRequest,
208 file: &ResourceFile,
209 task_resume: &mut TaskResume,
210 probe: Probe,
211 ) -> Result<FileOutcome> {
212 let key = file.url.as_str().to_string();
213
214 if let Some(stored) = task_resume.files.get(&key).cloned()
215 && stored.is_complete()
216 {
217 let final_path = manifest_path(&request.destination, &stored.final_path);
218 let part = manifest_path(&request.destination, &stored.part_path);
219 if tokio::fs::try_exists(&final_path).await? {
220 let size = tokio::fs::metadata(&final_path).await?.len();
221 let relative = relative_path(&request.destination, &final_path);
222 request.events.emit(EventKind::FileStarted {
223 task_id: request.task_id,
224 file: relative.clone(),
225 total_bytes: Some(size),
226 });
227 request.events.emit(EventKind::Progress {
228 task_id: request.task_id,
229 file: relative.clone(),
230 downloaded_bytes: size,
231 total_bytes: Some(size),
232 });
233 return Ok(FileOutcome::new(
234 final_path,
235 relative,
236 size,
237 stored.checksum.or(request.options.checksum.clone()),
238 false,
239 ));
240 }
241 if tokio::fs::try_exists(&part).await? {
242 ensure_parent(&final_path).await?;
243 remove_if_exists(&final_path).await?;
244 tokio::fs::rename(&part, &final_path).await?;
245 let size = tokio::fs::metadata(&final_path).await?.len();
246 let relative = relative_path(&request.destination, &final_path);
247 request.events.emit(EventKind::FileStarted {
248 task_id: request.task_id,
249 file: relative.clone(),
250 total_bytes: Some(size),
251 });
252 return Ok(FileOutcome::new(
253 final_path,
254 relative,
255 size,
256 stored.checksum.or(request.options.checksum.clone()),
257 false,
258 ));
259 }
260 }
261
262 let total = probe.total_size;
263 let validator = probe.validator();
264 let checksum = request
265 .options
266 .checksum
267 .clone()
268 .or_else(|| probe.checksum.clone())
269 .or_else(|| file.checksum.clone());
270 if request.options.verify == VerifyMode::Require && checksum.is_none() {
271 return Err(Error::ChecksumMissing);
272 }
273
274 let default_name = file
275 .path
276 .rsplit('/')
277 .next()
278 .map(sanitize_file_name)
279 .unwrap_or_else(|| "download".to_string());
280 let (mut final_path, relative) =
281 resolve_final_path(request, file, probe.filename.as_deref(), &default_name)?;
282
283 let stored = task_resume.files.get(&key).cloned();
284 let final_for_manifest = path_for_manifest(&request.destination, &final_path);
285 let stored_matches_path = stored
286 .as_ref()
287 .map(|stored| stored.final_path == final_for_manifest)
288 .unwrap_or(false);
289 let resume_valid = request.options.resume
290 && stored
291 .as_ref()
292 .map(|stored| {
293 stored_matches_path
294 && !stored.validator.is_empty()
295 && stored.validator.matches(&validator)
296 && match (stored.total_size, total) {
297 (Some(old), Some(new)) => old == new,
298 _ => true,
299 }
300 })
301 .unwrap_or(false);
302
303 if stored.is_some() && !resume_valid {
304 request.events.emit(EventKind::Warning {
305 task_id: Some(request.task_id),
306 message: "resume state is no longer valid; restarting this file".to_string(),
307 });
308 }
309
310 final_path = apply_conflict_policy(final_path, &request.options).await?;
311 let part_name = part_path(&final_path);
312 let part_for_manifest = path_for_manifest(&request.destination, &part_name);
313
314 let mut file_resume = FileResume {
315 url: key.clone(),
316 final_path: path_for_manifest(&request.destination, &final_path),
317 part_path: part_for_manifest.clone(),
318 validator: validator.clone(),
319 total_size: total,
320 segment_size: request.options.segment_size,
321 completed: if resume_valid {
322 stored
323 .as_ref()
324 .map(|stored| stored.completed.clone())
325 .unwrap_or_default()
326 } else {
327 Vec::new()
328 },
329 checksum: checksum.clone(),
330 metadata: {
331 let mut metadata = probe.headers.clone();
332 metadata.insert("final-url".to_string(), redacted_url(&probe.final_url));
333 metadata
334 },
335 updated_at_ms: now_ms(),
336 };
337
338 let existing_part_len = tokio::fs::metadata(&part_name)
340 .await
341 .ok()
342 .map(|metadata| metadata.len());
343 if !resume_valid {
344 file_resume.completed.clear();
345 } else if let Some(len) = existing_part_len {
346 let limit = total.map(|total| total.min(len)).unwrap_or(len);
347 file_resume.completed = shaul_core::range::clamp_to_len(&file_resume.completed, limit);
348 } else {
349 file_resume.completed.clear();
350 }
351 file_resume.normalize();
352
353 ensure_parent(&final_path).await?;
354 if let Some(total) = total {
355 let file = tokio::fs::OpenOptions::new()
356 .create(true)
357 .write(true)
358 .truncate(false)
359 .open(&part_name)
360 .await?;
361 if !resume_valid {
362 file.set_len(0).await?;
363 }
364 file.set_len(total).await?;
365 file.sync_all().await?;
366 } else if !resume_valid {
367 tokio::fs::OpenOptions::new()
368 .create(true)
369 .write(true)
370 .truncate(true)
371 .open(&part_name)
372 .await?;
373 } else if !tokio::fs::try_exists(&part_name).await? {
374 tokio::fs::File::create(&part_name).await?;
375 file_resume.completed.clear();
376 }
377
378 task_resume.files.insert(key.clone(), file_resume.clone());
379 request.checkpoint.save(task_resume).await?;
380
381 let initial = completed_bytes(&file_resume.completed);
382 let progress = Arc::new(AtomicU64::new(initial));
383 request.events.emit(EventKind::FileStarted {
384 task_id: request.task_id,
385 file: relative.clone(),
386 total_bytes: total,
387 });
388
389 let progress_stop = CancellationToken::new();
390 let reporter = spawn_progress_reporter(
391 request.events.clone(),
392 request.task_id,
393 relative.clone(),
394 total,
395 progress.clone(),
396 progress_stop.clone(),
397 );
398
399 let rate = self.request_rate(request);
400 let transfer_result = if let Some(0) = total {
401 task_resume.files.insert(key.clone(), file_resume.clone());
403 request.checkpoint.save(task_resume).await?;
404 Ok(file_resume.clone())
405 } else if probe.range_supported
406 && total.map(|total| total > 0).unwrap_or(false)
407 && request.options.connections > 1
408 {
409 self.transfer_segmented(
410 request,
411 file,
412 &part_name,
413 &key,
414 file_resume.clone(),
415 task_resume.clone(),
416 progress.clone(),
417 total,
418 &rate,
419 )
420 .await
421 } else {
422 self.transfer_single(
423 request,
424 file,
425 &part_name,
426 &key,
427 file_resume.clone(),
428 task_resume.clone(),
429 progress.clone(),
430 probe.range_supported,
431 &rate,
432 )
433 .await
434 };
435
436 progress_stop.cancel();
437 let _ = reporter.await;
438
439 let mut final_resume = match transfer_result {
440 Ok(resume) => resume,
441 Err(error) => return Err(error),
442 };
443
444 if let Some(total) = final_resume.total_size
445 && total > 0
446 {
447 final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
448 }
449 final_resume.normalize();
450 request.events.emit(EventKind::Progress {
451 task_id: request.task_id,
452 file: relative.clone(),
453 downloaded_bytes: final_resume.completed_bytes(),
454 total_bytes: final_resume.total_size,
455 });
456 task_resume.files.insert(key, final_resume.clone());
457 request.checkpoint.save(task_resume).await?;
458
459 ensure_parent(&final_path).await?;
460 remove_if_exists(&final_path).await?;
461 if tokio::fs::try_exists(&part_name).await? {
462 tokio::fs::rename(&part_name, &final_path).await?;
463 } else {
464 tokio::fs::write(&final_path, []).await?;
465 }
466 let size = tokio::fs::metadata(&final_path).await?.len();
467 Ok(FileOutcome::new(
468 final_path, relative, size, checksum, false,
469 ))
470 }
471
472 async fn probe(&self, request: &DownloadRequest, file: &ResourceFile) -> Result<Probe> {
473 let mut attempt = 0u32;
474 loop {
475 let headers = build_headers(&request.options, &file.headers, Some("bytes=0-0"), None)?;
476 let sent = self
477 .send_follow(&file.url, &headers, &request.options, &request.cancel)
478 .await;
479 match sent {
480 Err(Error::Cancelled) => return Err(Error::Cancelled),
481 Err(error) => {
482 if attempt >= request.options.retries {
483 return Err(error);
484 }
485 tracing::warn!(attempt, error = %error, "request failed; retrying");
486 retry_event(request, file, attempt + 1, error.to_string());
487 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
488 attempt += 1;
489 }
490 Ok((response, captured, final_url, _permit)) => {
491 let status = response.status();
492 let retry_after = retry_after(response.headers());
493 if status.is_server_error()
494 || status == StatusCode::REQUEST_TIMEOUT
495 || status == StatusCode::TOO_MANY_REQUESTS
496 {
497 drop(response);
498 if attempt >= request.options.retries {
499 return Err(http_error(status, &captured));
500 }
501 tracing::warn!(
502 attempt,
503 status = status.as_u16(),
504 "transient HTTP status; retrying"
505 );
506 retry_event(request, file, attempt + 1, format!("HTTP {status}"));
507 sleep_or_cancel(
508 retry_after.unwrap_or_else(|| backoff(attempt)),
509 &request.cancel,
510 )
511 .await?;
512 attempt += 1;
513 continue;
514 }
515 if matches!(
516 status,
517 StatusCode::UNAUTHORIZED
518 | StatusCode::FORBIDDEN
519 | StatusCode::NOT_FOUND
520 | StatusCode::GONE
521 ) {
522 return Err(http_error(status, &captured));
523 }
524
525 let content_range = captured
526 .get("content-range")
527 .and_then(|value| parse_content_range(value));
528 let linked_size = captured
529 .get("x-linked-size")
530 .and_then(|value| parse_u64(value));
531 let content_length = captured
532 .get("content-length")
533 .and_then(|value| parse_u64(value));
534 let (range_supported, total_size) = match status {
535 StatusCode::PARTIAL_CONTENT => (
536 true,
537 content_range
538 .and_then(|(_, _, total)| total)
539 .or(linked_size),
540 ),
541 StatusCode::OK => match content_range {
542 Some((_, _, total)) => (true, total.or(linked_size)),
548 None => (false, linked_size.or(content_length)),
549 },
550 StatusCode::RANGE_NOT_SATISFIABLE => {
551 (true, content_range.and_then(|(_, _, total)| total))
552 }
553 _ => {
554 return Err(Error::Backend(format!(
555 "unexpected HTTP status during probe: {status}"
556 )));
557 }
558 };
559
560 let checksum = captured
561 .get("x-linked-etag")
562 .and_then(|value| Checksum::from_etag(value))
563 .or_else(|| parse_digest(&captured));
564 let etag = captured
565 .get("etag")
566 .cloned()
567 .or_else(|| captured.get("x-linked-etag").cloned());
568 let last_modified = captured.get("last-modified").cloned();
569 let filename = captured
570 .get("content-disposition")
571 .and_then(|value| parse_content_disposition(value));
572 let mut headers = captured;
573 if let Some(total) = total_size {
574 headers.insert("total-size".to_string(), total.to_string());
575 }
576 return Ok(Probe {
577 final_url,
578 total_size,
579 range_supported,
580 filename,
581 etag,
582 last_modified,
583 checksum,
584 headers,
585 });
586 }
587 }
588 }
589 }
590
591 async fn send_follow(
592 &self,
593 url: &Url,
594 headers: &HeaderMap,
595 options: &shaul_core::ResolvedOptions,
596 cancel: &CancellationToken,
597 ) -> Result<(Response, BTreeMap<String, String>, Url, ConnectionPermit)> {
598 let mut current = url.clone();
599 let mut headers = headers.clone();
600 let mut captured = BTreeMap::new();
601
602 for _ in 0..MAX_REDIRECTS {
603 let permit = self
604 .budget
605 .acquire(current.host_str().unwrap_or_default(), cancel)
606 .await?;
607 let response = tokio::select! {
608 _ = cancel.cancelled() => return Err(Error::Cancelled),
609 response = self.client.get(current.clone()).headers(headers.clone()).send() => {
610 response.map_err(|error| Error::Backend(format!("request failed: {error}")))?
611 }
612 };
613 capture_headers(&response, &mut captured);
614 let status = response.status();
615 if matches!(
616 status,
617 StatusCode::MOVED_PERMANENTLY
618 | StatusCode::FOUND
619 | StatusCode::SEE_OTHER
620 | StatusCode::TEMPORARY_REDIRECT
621 | StatusCode::PERMANENT_REDIRECT
622 ) {
623 let location = response
624 .headers()
625 .get(LOCATION)
626 .and_then(|value| value.to_str().ok())
627 .ok_or_else(|| {
628 Error::Backend("redirect response missing Location".to_string())
629 })?;
630 let next = current
631 .join(location)
632 .map_err(|error| Error::Backend(format!("invalid redirect URL: {error}")))?;
633 if !same_origin(¤t, &next) && !options.request.allow_cross_origin_auth {
634 for name in options.request.sensitive_header_names() {
635 headers.remove(name.as_str());
636 }
637 headers.remove(AUTHORIZATION);
638 headers.remove(COOKIE);
639 }
640 current = next;
641 continue;
642 }
643 return Ok((response, captured, current, permit));
644 }
645 Err(Error::Backend("too many redirects".to_string()))
646 }
647
648 #[allow(clippy::too_many_arguments)]
649 async fn transfer_segmented(
650 &self,
651 request: &DownloadRequest,
652 file: &ResourceFile,
653 part_name: &Path,
654 key: &str,
655 file_resume: FileResume,
656 task_resume: TaskResume,
657 progress: Arc<AtomicU64>,
658 total: Option<u64>,
659 rate: &RateLimiters,
660 ) -> Result<FileResume> {
661 let total =
662 total.ok_or_else(|| Error::Backend("segmented transfer without size".to_string()))?;
663 let missing = missing_ranges(total, &file_resume.completed);
664 let mut chunks = Vec::new();
665 for range in missing {
666 chunks.extend(split_range(range, request.options.segment_size));
667 }
668 if chunks.is_empty() {
669 return Ok(file_resume);
670 }
671
672 let mut state = task_resume.clone();
673 state.files.insert(key.to_string(), file_resume.clone());
674 let resume_state = Arc::new(Mutex::new(state));
675 let checkpoint_lock = Arc::new(Mutex::new(()));
676 let queue = Arc::new(std::sync::Mutex::new(VecDeque::from(chunks)));
677 let range_broken = Arc::new(AtomicBool::new(false));
678 let child_cancel = request.cancel.child_token();
679 let if_range = strong_if_range(&file_resume.validator);
680 let workers = request
681 .options
682 .connections
683 .min(queue.lock().map(|queue| queue.len()).unwrap_or(1))
684 .max(1);
685
686 let mut set = tokio::task::JoinSet::new();
687 for _ in 0..workers {
688 let backend = self.clone();
689 let request = request.clone();
690 let file = file.clone();
691 let part_name = part_name.to_path_buf();
692 let key = key.to_string();
693 let queue = queue.clone();
694 let resume_state = resume_state.clone();
695 let checkpoint_lock = checkpoint_lock.clone();
696 let range_broken = range_broken.clone();
697 let progress = progress.clone();
698 let cancel = child_cancel.clone();
699 let if_range = if_range.clone();
700 let rate = rate.clone();
701 set.spawn(async move {
702 backend
703 .segment_worker(
704 &request,
705 &file,
706 &part_name,
707 &key,
708 queue,
709 resume_state,
710 checkpoint_lock,
711 range_broken,
712 progress,
713 cancel,
714 total,
715 if_range,
716 rate,
717 )
718 .await
719 });
720 }
721
722 let mut first_error = None;
723 while let Some(joined) = set.join_next().await {
724 match joined {
725 Ok(Ok(())) => {}
726 Ok(Err(FetchError::Cancelled)) => {}
727 Ok(Err(FetchError::RangeBroken)) => {}
728 Ok(Err(FetchError::Fatal(error))) => {
729 if first_error.is_none() {
730 first_error = Some(error);
731 }
732 child_cancel.cancel();
733 }
734 Ok(Err(FetchError::Retryable(message))) => {
735 if first_error.is_none() {
736 first_error = Some(Error::Backend(message));
737 }
738 child_cancel.cancel();
739 }
740 Err(error) => {
741 if first_error.is_none() {
742 first_error = Some(Error::Other(format!("segment worker failed: {error}")));
743 }
744 child_cancel.cancel();
745 }
746 }
747 }
748
749 if let Some(error) = first_error {
750 return Err(error);
751 }
752
753 if range_broken.load(Ordering::SeqCst) {
754 request.events.emit(EventKind::Warning {
755 task_id: Some(request.task_id),
756 message: "server ignored a ranged request; falling back to a single stream"
757 .to_string(),
758 });
759 let file_handle = tokio::fs::OpenOptions::new()
760 .create(true)
761 .write(true)
762 .truncate(false)
763 .open(part_name)
764 .await?;
765 file_handle.set_len(0).await?;
766 file_handle.sync_all().await?;
767 progress.store(0, Ordering::Relaxed);
768 let mut reset = file_resume;
769 reset.completed.clear();
770 reset.normalize();
771 return self
772 .transfer_single(
773 request,
774 file,
775 part_name,
776 key,
777 reset,
778 task_resume,
779 progress,
780 false,
781 rate,
782 )
783 .await;
784 }
785
786 let final_state = resume_state.lock().await.clone();
787 let mut final_resume = final_state
788 .files
789 .get(key)
790 .cloned()
791 .ok_or_else(|| Error::Backend("missing resume state after transfer".to_string()))?;
792 final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
793 final_resume.normalize();
794 let mut snapshot = final_state;
795 snapshot.files.insert(key.to_string(), final_resume.clone());
796 request.checkpoint.save(&snapshot).await?;
797 Ok(final_resume)
798 }
799
800 #[allow(clippy::too_many_arguments)]
801 async fn segment_worker(
802 &self,
803 request: &DownloadRequest,
804 file: &ResourceFile,
805 part_name: &Path,
806 key: &str,
807 queue: Arc<std::sync::Mutex<VecDeque<shaul_core::ByteRange>>>,
808 resume_state: Arc<Mutex<TaskResume>>,
809 checkpoint_lock: Arc<Mutex<()>>,
810 range_broken: Arc<AtomicBool>,
811 progress: Arc<AtomicU64>,
812 cancel: CancellationToken,
813 total: u64,
814 if_range: Option<String>,
815 rate: RateLimiters,
816 ) -> std::result::Result<(), FetchError> {
817 let mut part = tokio::fs::OpenOptions::new()
818 .write(true)
819 .open(part_name)
820 .await
821 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
822 loop {
823 if cancel.is_cancelled() || range_broken.load(Ordering::SeqCst) {
824 break;
825 }
826 let range = {
827 let mut queue = queue
828 .lock()
829 .map_err(|_| FetchError::Fatal(Error::Other("work queue poisoned".into())))?;
830 queue.pop_front()
831 };
832 let Some(range) = range else {
833 break;
834 };
835 match self
836 .fetch_range(
837 request,
838 file,
839 range,
840 &mut part,
841 &progress,
842 &cancel,
843 total,
844 if_range.as_deref(),
845 &rate,
846 )
847 .await
848 {
849 Ok(()) => {
850 let _guard = checkpoint_lock.lock().await;
851 let snapshot = {
852 let mut state = resume_state.lock().await;
853 if let Some(entry) = state.files.get_mut(key) {
854 entry.completed.push(range);
855 entry.normalize();
856 }
857 state.clone()
858 };
859 request
860 .checkpoint
861 .save(&snapshot)
862 .await
863 .map_err(FetchError::Fatal)?;
864 }
865 Err(FetchError::Cancelled) => break,
866 Err(FetchError::RangeBroken) => {
867 range_broken.store(true, Ordering::SeqCst);
868 cancel.cancel();
869 break;
870 }
871 Err(error) => {
872 cancel.cancel();
873 return Err(error);
874 }
875 }
876 }
877 part.flush()
878 .await
879 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
880 Ok(())
881 }
882
883 #[allow(clippy::too_many_arguments)]
884 async fn fetch_range(
885 &self,
886 request: &DownloadRequest,
887 file: &ResourceFile,
888 range: shaul_core::ByteRange,
889 part: &mut tokio::fs::File,
890 progress: &AtomicU64,
891 cancel: &CancellationToken,
892 total: u64,
893 if_range: Option<&str>,
894 rate: &RateLimiters,
895 ) -> std::result::Result<(), FetchError> {
896 let mut attempt = 0u32;
897 loop {
898 if cancel.is_cancelled() {
899 return Err(FetchError::Cancelled);
900 }
901 let range_header = format!("bytes={}-{}", range.start, range.end - 1);
902 let headers = build_headers(
903 &request.options,
904 &file.headers,
905 Some(&range_header),
906 if_range,
907 )
908 .map_err(FetchError::Fatal)?;
909 let sent = self
910 .send_follow(&file.url, &headers, &request.options, cancel)
911 .await;
912 let (response, captured, _, _permit) = match sent {
913 Err(Error::Cancelled) => return Err(FetchError::Cancelled),
914 Err(error) => {
915 if attempt >= request.options.retries {
916 return Err(FetchError::Fatal(error));
917 }
918 tracing::warn!(attempt, error = %error, "transfer request failed; retrying");
919 retry_event(request, file, attempt + 1, error.to_string());
920 sleep_or_cancel(backoff(attempt), cancel)
921 .await
922 .map_err(FetchError::Fatal)?;
923 attempt += 1;
924 continue;
925 }
926 Ok(response) => response,
927 };
928 let status = response.status();
929 let retry_after = retry_after(response.headers());
930
931 if status == StatusCode::RANGE_NOT_SATISFIABLE {
932 return Err(FetchError::Fatal(Error::Backend(format!(
933 "range {} is not satisfiable",
934 range_header
935 ))));
936 }
937 if status.is_server_error()
938 || status == StatusCode::REQUEST_TIMEOUT
939 || status == StatusCode::TOO_MANY_REQUESTS
940 {
941 drop(response);
942 if attempt >= request.options.retries {
943 return Err(FetchError::Fatal(http_error(status, &captured)));
944 }
945 tracing::warn!(
946 attempt,
947 status = status.as_u16(),
948 "transient HTTP status during transfer; retrying"
949 );
950 retry_event(request, file, attempt + 1, format!("HTTP {status}"));
951 sleep_or_cancel(retry_after.unwrap_or_else(|| backoff(attempt)), cancel)
952 .await
953 .map_err(FetchError::Fatal)?;
954 attempt += 1;
955 continue;
956 }
957
958 let content_range = captured
959 .get("content-range")
960 .and_then(|value| parse_content_range(value));
961 if status == StatusCode::OK && content_range.is_none() {
962 return Err(FetchError::RangeBroken);
963 }
964 let is_partial = status == StatusCode::PARTIAL_CONTENT
965 || (status == StatusCode::OK && content_range.is_some());
966 if !is_partial {
967 return Err(FetchError::Fatal(http_error(status, &captured)));
968 }
969 let parsed = content_range.ok_or_else(|| {
970 FetchError::Fatal(Error::Backend(
971 "ranged response missing a valid Content-Range".to_string(),
972 ))
973 })?;
974 if parsed.0 != range.start || parsed.1 + 1 != range.end {
975 return Err(FetchError::Fatal(Error::Backend(format!(
976 "server returned range {}-{}, expected {}-{}",
977 parsed.0,
978 parsed.1,
979 range.start,
980 range.end - 1
981 ))));
982 }
983 if let Some(server_total) = parsed.2
984 && server_total != total
985 {
986 return Err(FetchError::Fatal(Error::Backend(format!(
987 "file size changed from {total} to {server_total}"
988 ))));
989 }
990
991 part.seek(SeekFrom::Start(range.start))
992 .await
993 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
994 let mut written = 0u64;
995 let mut stream = response.bytes_stream();
996 let retry_message;
997 loop {
998 let next = tokio::select! {
999 _ = cancel.cancelled() => return Err(FetchError::Cancelled),
1000 next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1001 };
1002 match next {
1003 Ok(Some(Ok(chunk))) => {
1004 if written + chunk.len() as u64 > range.len() {
1005 retry_message =
1006 Some("server sent more data than requested".to_string());
1007 break;
1008 }
1009 rate.throttle(chunk.len() as u64, cancel)
1010 .await
1011 .map_err(FetchError::Fatal)?;
1012 part.write_all(&chunk)
1013 .await
1014 .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
1015 written += chunk.len() as u64;
1016 progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1017 }
1018 Ok(Some(Err(error))) => {
1019 retry_message = Some(format!("stream error: {error}"));
1020 break;
1021 }
1022 Ok(None) => {
1023 if written == range.len() {
1024 return Ok(());
1025 }
1026 retry_message = Some(format!(
1027 "short ranged response: {written} of {} bytes",
1028 range.len()
1029 ));
1030 break;
1031 }
1032 Err(_) => {
1033 retry_message = Some("read timed out".to_string());
1034 break;
1035 }
1036 }
1037 }
1038
1039 let current = progress.load(Ordering::Relaxed);
1042 progress.fetch_sub(written.min(current), Ordering::Relaxed);
1043 let message = retry_message.unwrap_or_else(|| "range transfer failed".to_string());
1044 if attempt >= request.options.retries {
1045 return Err(FetchError::Retryable(message));
1046 }
1047 retry_event(request, file, attempt + 1, message);
1048 sleep_or_cancel(backoff(attempt), cancel)
1049 .await
1050 .map_err(FetchError::Fatal)?;
1051 attempt += 1;
1052 }
1053 }
1054
1055 #[allow(clippy::too_many_arguments)]
1056 async fn transfer_single(
1057 &self,
1058 request: &DownloadRequest,
1059 file: &ResourceFile,
1060 part_name: &Path,
1061 key: &str,
1062 mut file_resume: FileResume,
1063 mut task_resume: TaskResume,
1064 progress: Arc<AtomicU64>,
1065 range_supported: bool,
1066 rate: &RateLimiters,
1067 ) -> Result<FileResume> {
1068 let mut offset = if request.options.resume && range_supported {
1069 contiguous_end(&file_resume.completed)
1070 } else {
1071 0
1072 };
1073 if file_resume
1074 .total_size
1075 .map(|total| offset >= total)
1076 .unwrap_or(false)
1077 {
1078 return Ok(file_resume);
1079 }
1080
1081 let mut part = tokio::fs::OpenOptions::new()
1082 .create(true)
1083 .write(true)
1084 .truncate(false)
1085 .open(part_name)
1086 .await?;
1087 if offset == 0 {
1088 part.set_len(0).await?;
1089 }
1090 part.seek(SeekFrom::Start(offset)).await?;
1091
1092 let mut attempt = 0u32;
1093 loop {
1094 if request.cancel.is_cancelled() {
1095 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1096 return Err(Error::Cancelled);
1097 }
1098 let range = if offset > 0 && range_supported {
1099 Some(format!("bytes={offset}-"))
1100 } else {
1101 None
1102 };
1103 let if_range = if offset > 0 {
1104 strong_if_range(&file_resume.validator)
1105 } else {
1106 None
1107 };
1108 let headers = build_headers(
1109 &request.options,
1110 &file.headers,
1111 range.as_deref(),
1112 if_range.as_deref(),
1113 )?;
1114 let sent = self
1115 .send_follow(&file.url, &headers, &request.options, &request.cancel)
1116 .await;
1117 let (response, captured, _, _permit) = match sent {
1118 Err(Error::Cancelled) => {
1119 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
1120 .await?;
1121 return Err(Error::Cancelled);
1122 }
1123 Err(error) => {
1124 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
1125 .await?;
1126 if attempt >= request.options.retries {
1127 return Err(error);
1128 }
1129 retry_event(request, file, attempt + 1, error.to_string());
1130 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1131 attempt += 1;
1132 continue;
1133 }
1134 Ok(response) => response,
1135 };
1136 let status = response.status();
1137 let retry_after = retry_after(response.headers());
1138
1139 let content_range = captured
1140 .get("content-range")
1141 .and_then(|value| parse_content_range(value));
1142 if status.is_server_error()
1143 || status == StatusCode::REQUEST_TIMEOUT
1144 || status == StatusCode::TOO_MANY_REQUESTS
1145 {
1146 drop(response);
1147 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1148 if attempt >= request.options.retries {
1149 return Err(http_error(status, &captured));
1150 }
1151 retry_event(request, file, attempt + 1, format!("HTTP {status}"));
1152 sleep_or_cancel(
1153 retry_after.unwrap_or_else(|| backoff(attempt)),
1154 &request.cancel,
1155 )
1156 .await?;
1157 attempt += 1;
1158 continue;
1159 }
1160
1161 let is_partial = status == StatusCode::PARTIAL_CONTENT
1162 || (status == StatusCode::OK && content_range.is_some());
1163 if is_partial {
1164 if let Some((start, _, _)) = content_range
1165 && offset > 0
1166 && start != offset
1167 {
1168 return Err(Error::Backend(format!(
1169 "server resumed at {start}, expected {offset}"
1170 )));
1171 }
1172 } else if status == StatusCode::OK {
1173 if offset > 0 {
1174 part.set_len(0).await?;
1175 part.seek(SeekFrom::Start(0)).await?;
1176 offset = 0;
1177 progress.store(0, Ordering::Relaxed);
1178 }
1179 } else {
1180 return Err(http_error(status, &captured));
1181 }
1182
1183 part.seek(SeekFrom::Start(offset)).await?;
1184 let mut stream = response.bytes_stream();
1185 let mut since_checkpoint = 0u64;
1186 let retry_message;
1187 loop {
1188 let next = tokio::select! {
1189 _ = request.cancel.cancelled() => {
1190 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1191 part.sync_all().await?;
1192 return Err(Error::Cancelled);
1193 }
1194 next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1195 };
1196 match next {
1197 Ok(Some(Ok(chunk))) => {
1198 if let Some(total) = file_resume.total_size
1199 && offset + chunk.len() as u64 > total
1200 {
1201 return Err(Error::Backend(
1202 "server sent more data than the known file size".to_string(),
1203 ));
1204 }
1205 rate.throttle(chunk.len() as u64, &request.cancel).await?;
1206 part.write_all(&chunk).await?;
1207 offset += chunk.len() as u64;
1208 progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1209 since_checkpoint += chunk.len() as u64;
1210 if since_checkpoint >= request.options.segment_size {
1211 checkpoint_single(
1212 request,
1213 &mut task_resume,
1214 key,
1215 &mut file_resume,
1216 offset,
1217 )
1218 .await?;
1219 since_checkpoint = 0;
1220 }
1221 }
1222 Ok(Some(Err(error))) => {
1223 retry_message = Some(format!("stream error: {error}"));
1224 break;
1225 }
1226 Ok(None) => {
1227 if let Some(total) = file_resume.total_size
1228 && offset < total
1229 {
1230 retry_message =
1231 Some(format!("short response: {offset} of {total} bytes"));
1232 break;
1233 }
1234 part.flush().await?;
1235 part.sync_all().await?;
1236 file_resume.total_size = file_resume.total_size.or(Some(offset));
1237 file_resume.completed = if offset > 0 {
1238 vec![shaul_core::ByteRange::new(0, offset)?]
1239 } else {
1240 Vec::new()
1241 };
1242 file_resume.normalize();
1243 task_resume
1244 .files
1245 .insert(key.to_string(), file_resume.clone());
1246 request.checkpoint.save(&task_resume).await?;
1247 return Ok(file_resume);
1248 }
1249 Err(_) => {
1250 retry_message = Some("read timed out".to_string());
1251 break;
1252 }
1253 }
1254 }
1255
1256 part.flush().await?;
1257 checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1258 let message = retry_message.unwrap_or_else(|| "download failed".to_string());
1259 if attempt >= request.options.retries {
1260 return Err(Error::Backend(message));
1261 }
1262 retry_event(request, file, attempt + 1, message);
1263 sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1264 attempt += 1;
1265 }
1266 }
1267}
1268
1269#[async_trait]
1270impl DownloadBackend for HttpBackend {
1271 fn name(&self) -> &'static str {
1272 "http"
1273 }
1274
1275 fn capabilities(&self) -> BackendCapabilities {
1276 BackendCapabilities::new(true, true, true)
1277 }
1278
1279 fn supports(&self, resource: &Resource) -> bool {
1280 matches!(&resource.kind, ResourceKind::Http)
1281 }
1282
1283 #[tracing::instrument(
1284 name = "shaul.http.download",
1285 level = "debug",
1286 skip(self, request),
1287 fields(task_id = %request.task_id, files = request.resource.files.len())
1288 )]
1289 async fn download(&self, request: DownloadRequest) -> Result<DownloadOutcome> {
1290 if request.options.file_jobs > 1 && request.resource.files.len() > 1 {
1291 self.download_concurrent(&request).await
1292 } else {
1293 self.download_sequential(&request).await
1294 }
1295 }
1296}
1297
1298impl HttpBackend {
1299 async fn download_sequential(&self, request: &DownloadRequest) -> Result<DownloadOutcome> {
1300 let mut task_resume = request.resume.clone().unwrap_or_default();
1301 let mut outcomes = Vec::new();
1302 for file in &request.resource.files {
1303 if request.cancel.is_cancelled() {
1304 return Err(Error::Cancelled);
1305 }
1306 let outcome = self.download_file(request, file, &mut task_resume).await?;
1307 outcomes.push(outcome);
1308 request.checkpoint.save(&task_resume).await?;
1309 }
1310 Ok(DownloadOutcome::Completed(outcomes))
1311 }
1312
1313 async fn download_concurrent(&self, request: &DownloadRequest) -> Result<DownloadOutcome> {
1314 let shared = Arc::new(Mutex::new(request.resume.clone().unwrap_or_default()));
1315 let cancel = request.cancel.child_token();
1316 let semaphore = Arc::new(tokio::sync::Semaphore::new(
1317 request.options.file_jobs.max(1),
1318 ));
1319 let mut tasks = tokio::task::JoinSet::new();
1320 for (index, file) in request.resource.files.iter().cloned().enumerate() {
1321 let key = file.url.as_str().to_string();
1322 let mut local = TaskResume::default();
1323 if let Some(stored) = shared.lock().await.files.get(&key).cloned() {
1324 local.files.insert(key, stored);
1325 }
1326 let mut file_request = request.clone();
1327 file_request.cancel = cancel.clone();
1328 let checkpoint = Arc::new(FileCheckpoint {
1329 shared: shared.clone(),
1330 inner: request.checkpoint.clone(),
1331 });
1332 file_request.checkpoint = checkpoint.clone();
1333 let backend = self.clone();
1334 let permit = semaphore.clone();
1335 tasks.spawn(async move {
1336 let _permit = permit
1337 .acquire_owned()
1338 .await
1339 .map_err(|_| Error::Other("file job semaphore closed".to_string()))?;
1340 let mut local = local;
1341 let outcome = backend
1342 .download_file(&file_request, &file, &mut local)
1343 .await?;
1344 checkpoint.save(&local).await?;
1345 Ok::<_, Error>((index, outcome))
1346 });
1347 }
1348
1349 let mut outcomes: Vec<Option<FileOutcome>> =
1350 (0..request.resource.files.len()).map(|_| None).collect();
1351 let mut first_error = None;
1352 while let Some(joined) = tasks.join_next().await {
1353 match joined {
1354 Ok(Ok((index, outcome))) => outcomes[index] = Some(outcome),
1355 Ok(Err(error)) => {
1356 first_error.get_or_insert(error);
1357 cancel.cancel();
1358 }
1359 Err(error) => {
1360 first_error
1361 .get_or_insert_with(|| Error::Other(format!("file job failed: {error}")));
1362 cancel.cancel();
1363 }
1364 }
1365 }
1366 if let Some(error) = first_error {
1367 return Err(error);
1368 }
1369 let outcomes = outcomes
1370 .into_iter()
1371 .map(|outcome| outcome.ok_or_else(|| Error::Other("missing file outcome".to_string())))
1372 .collect::<Result<Vec<_>>>()?;
1373 Ok(DownloadOutcome::Completed(outcomes))
1374 }
1375}
1376
1377struct FileCheckpoint {
1378 shared: Arc<Mutex<TaskResume>>,
1379 inner: Arc<dyn ResumeCheckpoint>,
1380}
1381
1382#[async_trait]
1383impl ResumeCheckpoint for FileCheckpoint {
1384 async fn save(&self, resume: &TaskResume) -> Result<()> {
1385 let mut shared = self.shared.lock().await;
1386 for (key, file) in &resume.files {
1387 shared.files.insert(key.clone(), file.clone());
1388 }
1389 self.inner.save(&shared).await
1390 }
1391}
1392
1393fn warning(request: &DownloadRequest, message: impl Into<String>) {
1394 request.events.emit(EventKind::Warning {
1395 task_id: Some(request.task_id),
1396 message: message.into(),
1397 });
1398}
1399
1400fn retry_event(
1401 request: &DownloadRequest,
1402 file: &ResourceFile,
1403 attempt: u32,
1404 error: impl Into<String>,
1405) {
1406 request.events.emit(EventKind::Retry {
1407 task_id: request.task_id,
1408 file: file.path.clone(),
1409 attempt,
1410 error: error.into(),
1411 });
1412}
1413
1414fn mirror_compatible(reference: &Validator, mirror: &Validator, checksum_known: bool) -> bool {
1415 match (reference.size, mirror.size) {
1416 (Some(reference), Some(mirror)) if reference != mirror => return false,
1417 (None, _) | (_, None) if !checksum_known => return false,
1418 _ => {}
1419 }
1420 if !checksum_known
1421 && let (Some(reference), Some(mirror)) = (&reference.etag, &mirror.etag)
1422 && !reference.starts_with("W/")
1423 && !mirror.starts_with("W/")
1424 && reference != mirror
1425 {
1426 return false;
1427 }
1428 true
1429}
1430
1431struct Probe {
1432 final_url: Url,
1433 total_size: Option<u64>,
1434 range_supported: bool,
1435 filename: Option<String>,
1436 etag: Option<String>,
1437 last_modified: Option<String>,
1438 checksum: Option<Checksum>,
1439 headers: BTreeMap<String, String>,
1440}
1441
1442impl Probe {
1443 fn validator(&self) -> Validator {
1444 Validator {
1445 etag: self.etag.clone(),
1446 last_modified: self.last_modified.clone(),
1447 size: self.total_size,
1448 }
1449 }
1450}
1451
1452#[derive(Debug)]
1453enum FetchError {
1454 RangeBroken,
1455 Cancelled,
1456 Retryable(String),
1457 Fatal(Error),
1458}
1459
1460fn build_headers(
1461 options: &shaul_core::ResolvedOptions,
1462 file_headers: &[Header],
1463 range: Option<&str>,
1464 if_range: Option<&str>,
1465) -> Result<HeaderMap> {
1466 let mut headers = HeaderMap::new();
1467 for header in &options.request.headers {
1468 if !is_reserved(&header.name) {
1469 insert_header(
1470 &mut headers,
1471 &header.name,
1472 &shaul_core::expand_env(&header.value)?,
1473 )?;
1474 }
1475 }
1476 for header in file_headers {
1477 if !is_reserved(&header.name) {
1478 insert_header(
1479 &mut headers,
1480 &header.name,
1481 &shaul_core::expand_env(&header.value)?,
1482 )?;
1483 }
1484 }
1485
1486 if !headers.contains_key(AUTHORIZATION)
1487 && let Some(auth) = &options.request.auth
1488 {
1489 match auth {
1490 shaul_core::Auth::Bearer { token } => {
1491 insert_header(
1492 &mut headers,
1493 "authorization",
1494 &format!("Bearer {}", shaul_core::expand_env(token)?),
1495 )?;
1496 }
1497 shaul_core::Auth::Basic { username, password } => {
1498 let encoded = base64::engine::general_purpose::STANDARD.encode(format!(
1499 "{}:{}",
1500 shaul_core::expand_env(username)?,
1501 shaul_core::expand_env(password)?
1502 ));
1503 insert_header(&mut headers, "authorization", &format!("Basic {encoded}"))?;
1504 }
1505 shaul_core::Auth::Header(header) => {
1506 insert_header(
1507 &mut headers,
1508 &header.name,
1509 &shaul_core::expand_env(&header.value)?,
1510 )?;
1511 }
1512 }
1513 }
1514
1515 headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
1516 insert_header(&mut headers, "user-agent", &options.user_agent)?;
1517 if let Some(range) = range {
1518 insert_header(&mut headers, "range", range)?;
1519 }
1520 if let Some(if_range) = if_range {
1521 insert_header(&mut headers, "if-range", if_range)?;
1522 }
1523 Ok(headers)
1524}
1525
1526fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<()> {
1527 let name = HeaderName::from_bytes(name.as_bytes())
1528 .map_err(|_| Error::Config(format!("invalid header name: {name}")))?;
1529 let value = HeaderValue::from_str(value)
1530 .map_err(|_| Error::Config(format!("invalid value for header: {name}")))?;
1531 headers.insert(name, value);
1532 Ok(())
1533}
1534
1535fn is_reserved(name: &str) -> bool {
1536 RESERVED_HEADERS
1537 .iter()
1538 .any(|reserved| name.eq_ignore_ascii_case(reserved))
1539}
1540
1541fn capture_headers(response: &Response, captured: &mut BTreeMap<String, String>) {
1542 for name in CAPTURE_HEADERS {
1543 if let Some(value) = response.headers().get(*name)
1544 && let Ok(value) = value.to_str()
1545 {
1546 captured.insert((*name).to_string(), value.to_string());
1547 }
1548 }
1549}
1550
1551fn redacted_url(url: &Url) -> String {
1552 let mut url = url.clone();
1553 url.set_query(None);
1554 url.set_fragment(None);
1555 url.to_string()
1556}
1557
1558fn same_origin(left: &Url, right: &Url) -> bool {
1559 left.scheme() == right.scheme()
1560 && left.host_str() == right.host_str()
1561 && left.port_or_known_default() == right.port_or_known_default()
1562}
1563
1564fn parse_content_range(value: &str) -> Option<(u64, u64, Option<u64>)> {
1565 let rest = value.trim().strip_prefix("bytes")?.trim();
1566 let (range, total) = rest.split_once('/')?;
1567 let total = if total.trim() == "*" {
1568 None
1569 } else {
1570 Some(total.trim().parse::<u64>().ok()?)
1571 };
1572 if range.trim() == "*" {
1573 return Some((0, 0, total));
1574 }
1575 let (start, end) = range.split_once('-')?;
1576 Some((
1577 start.trim().parse::<u64>().ok()?,
1578 end.trim().parse::<u64>().ok()?,
1579 total,
1580 ))
1581}
1582
1583fn parse_u64(value: &str) -> Option<u64> {
1584 value.trim().parse::<u64>().ok()
1585}
1586
1587fn retry_after(headers: &HeaderMap) -> Option<Duration> {
1588 headers
1589 .get("retry-after")
1590 .and_then(|value| value.to_str().ok())
1591 .and_then(|value| value.trim().parse::<u64>().ok())
1592 .map(|seconds| Duration::from_secs(seconds.min(60)))
1593}
1594
1595fn backoff(attempt: u32) -> Duration {
1596 let base = 250u64.saturating_mul(1u64 << attempt.min(5));
1597 let jitter = SystemTime::now()
1598 .duration_since(UNIX_EPOCH)
1599 .map(|elapsed| u64::from(elapsed.subsec_nanos()) % (base / 4 + 1))
1600 .unwrap_or(0);
1601 Duration::from_millis(base.saturating_add(jitter))
1602}
1603
1604async fn sleep_or_cancel(duration: Duration, cancel: &CancellationToken) -> Result<()> {
1605 tokio::select! {
1606 _ = cancel.cancelled() => Err(Error::Cancelled),
1607 _ = tokio::time::sleep(duration) => Ok(()),
1608 }
1609}
1610
1611fn proxy_from_config(config: &Config) -> Result<Option<reqwest::Proxy>> {
1612 let Some(url) = config.network.proxy.as_deref() else {
1613 return Ok(None);
1614 };
1615 let mut proxy = reqwest::Proxy::all(url)
1616 .map_err(|error| Error::Config(format!("invalid proxy URL: {error}")))?;
1617 if let Some(value) = config.network.no_proxy.as_deref()
1618 && let Some(no_proxy) = reqwest::NoProxy::from_string(value)
1619 {
1620 proxy = proxy.no_proxy(Some(no_proxy));
1621 }
1622 Ok(Some(proxy))
1623}
1624
1625fn http_error(status: StatusCode, captured: &BTreeMap<String, String>) -> Error {
1626 let code = captured
1627 .get("x-error-code")
1628 .map(|code| format!(" ({code})"))
1629 .unwrap_or_default();
1630 Error::Backend(format!("HTTP {status}{code}"))
1631}
1632
1633fn parse_digest(captured: &BTreeMap<String, String>) -> Option<Checksum> {
1634 if let Some(value) = captured.get("digest") {
1635 for part in value.split(',') {
1636 let Some((algorithm, encoded)) = part.trim().split_once('=') else {
1637 continue;
1638 };
1639 let algorithm = match algorithm.trim().to_ascii_lowercase().as_str() {
1640 "sha-256" | "sha256" => ChecksumAlgorithm::Sha256,
1641 "sha-512" | "sha512" => ChecksumAlgorithm::Sha512,
1642 "sha" | "sha-1" | "sha1" => ChecksumAlgorithm::Sha1,
1643 "md5" => ChecksumAlgorithm::Md5,
1644 _ => continue,
1645 };
1646 if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(encoded.trim())
1647 && let Ok(checksum) = Checksum::new(algorithm, hex::encode(bytes))
1648 {
1649 return Some(checksum);
1650 }
1651 }
1652 }
1653 if let Some(value) = captured.get("content-md5")
1654 && let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(value.trim())
1655 && let Ok(checksum) = Checksum::new(ChecksumAlgorithm::Md5, hex::encode(bytes))
1656 {
1657 return Some(checksum);
1658 }
1659 None
1660}
1661
1662fn strong_if_range(validator: &Validator) -> Option<String> {
1663 if let Some(etag) = &validator.etag
1664 && !etag.starts_with("W/")
1665 {
1666 return Some(etag.clone());
1667 }
1668 validator.last_modified.clone()
1669}
1670
1671fn contiguous_end(completed: &[shaul_core::ByteRange]) -> u64 {
1672 let merged = merge_ranges(completed.to_vec());
1673 if merged.len() == 1 && merged[0].start == 0 {
1674 merged[0].end
1675 } else {
1676 0
1677 }
1678}
1679
1680fn manifest_path(destination: &Path, value: &str) -> PathBuf {
1681 let path = Path::new(value);
1682 if path.is_absolute() {
1683 path.to_path_buf()
1684 } else {
1685 destination.join(path)
1686 }
1687}
1688
1689fn path_for_manifest(destination: &Path, path: &Path) -> String {
1690 match path.strip_prefix(destination) {
1691 Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1692 Err(_) => path.to_string_lossy().to_string(),
1693 }
1694}
1695
1696fn relative_path(destination: &Path, path: &Path) -> String {
1697 match path.strip_prefix(destination) {
1698 Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1699 Err(_) => path
1700 .file_name()
1701 .map(|name| name.to_string_lossy().to_string())
1702 .unwrap_or_else(|| "download".to_string()),
1703 }
1704}
1705
1706fn resolve_final_path(
1707 request: &DownloadRequest,
1708 file: &ResourceFile,
1709 probe_filename: Option<&str>,
1710 default_name: &str,
1711) -> Result<(PathBuf, String)> {
1712 let single = request.resource.files.len() == 1;
1713 let response_name = if file.use_response_filename {
1714 probe_filename.map(sanitize_file_name)
1715 } else {
1716 None
1717 };
1718 let name = response_name
1719 .clone()
1720 .unwrap_or_else(|| default_name.to_string());
1721
1722 if let Some(output) = &request.options.output {
1723 if single {
1724 if output.is_dir() {
1725 let relative = if file.use_response_filename {
1726 name.clone()
1727 } else {
1728 file.path.clone()
1729 };
1730 let path = output.join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1731 return Ok((path.clone(), relative_path(&request.destination, &path)));
1732 }
1733 return Ok((output.clone(), relative_path(&request.destination, output)));
1734 }
1735 let path = output.join(file.path.replace('/', std::path::MAIN_SEPARATOR_STR));
1736 return Ok((path.clone(), relative_path(&request.destination, &path)));
1737 }
1738
1739 let relative = if single && file.use_response_filename {
1740 name
1741 } else {
1742 file.path.clone()
1743 };
1744 let path = request
1745 .destination
1746 .join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1747 Ok((path.clone(), relative_path(&request.destination, &path)))
1748}
1749
1750async fn apply_conflict_policy(
1751 path: PathBuf,
1752 options: &shaul_core::ResolvedOptions,
1753) -> Result<PathBuf> {
1754 if !tokio::fs::try_exists(&path).await? {
1755 return Ok(path);
1756 }
1757 if options.overwrite {
1758 remove_if_exists(&path).await?;
1759 return Ok(path);
1760 }
1761 if options.rename_on_conflict {
1762 return unique_path(&path).await;
1763 }
1764 Err(Error::Backend(format!(
1765 "destination already exists: {} (use --force or --rename)",
1766 path.display()
1767 )))
1768}
1769
1770async fn unique_path(path: &Path) -> Result<PathBuf> {
1771 let stem = path
1772 .file_stem()
1773 .map(|stem| stem.to_string_lossy().to_string())
1774 .unwrap_or_else(|| "download".to_string());
1775 let extension = path
1776 .extension()
1777 .map(|extension| format!(".{}", extension.to_string_lossy()))
1778 .unwrap_or_default();
1779 for index in 1..10_000 {
1780 let candidate = path.with_file_name(format!("{stem}-{index}{extension}"));
1781 if !tokio::fs::try_exists(&candidate).await? {
1782 return Ok(candidate);
1783 }
1784 }
1785 Err(Error::Backend(
1786 "could not find a free destination name".to_string(),
1787 ))
1788}
1789
1790fn now_ms() -> u64 {
1791 SystemTime::now()
1792 .duration_since(UNIX_EPOCH)
1793 .map(|duration| duration.as_millis() as u64)
1794 .unwrap_or(0)
1795}
1796
1797async fn ensure_parent(path: &Path) -> Result<()> {
1798 if let Some(parent) = path.parent()
1799 && !parent.as_os_str().is_empty()
1800 {
1801 tokio::fs::create_dir_all(parent).await?;
1802 }
1803 Ok(())
1804}
1805
1806async fn remove_if_exists(path: &Path) -> Result<()> {
1807 match tokio::fs::remove_file(path).await {
1808 Ok(()) => Ok(()),
1809 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1810 Err(error) => Err(Error::Io(error)),
1811 }
1812}
1813
1814async fn checkpoint_single(
1815 request: &DownloadRequest,
1816 task_resume: &mut TaskResume,
1817 key: &str,
1818 file_resume: &mut FileResume,
1819 offset: u64,
1820) -> Result<()> {
1821 file_resume.completed = if offset > 0 {
1822 vec![shaul_core::ByteRange::new(0, offset)?]
1823 } else {
1824 Vec::new()
1825 };
1826 file_resume.normalize();
1827 task_resume
1828 .files
1829 .insert(key.to_string(), file_resume.clone());
1830 request.checkpoint.save(task_resume).await
1831}
1832
1833fn spawn_progress_reporter(
1834 events: EventSink,
1835 task_id: TaskId,
1836 file: String,
1837 total: Option<u64>,
1838 downloaded: Arc<AtomicU64>,
1839 stop: CancellationToken,
1840) -> tokio::task::JoinHandle<()> {
1841 tokio::spawn(async move {
1842 let mut ticker = tokio::time::interval(Duration::from_millis(100));
1843 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1844 loop {
1845 tokio::select! {
1846 _ = ticker.tick() => {
1847 events.emit(EventKind::Progress {
1848 task_id,
1849 file: file.clone(),
1850 downloaded_bytes: downloaded.load(Ordering::Relaxed),
1851 total_bytes: total,
1852 });
1853 }
1854 _ = stop.cancelled() => {
1855 events.emit(EventKind::Progress {
1856 task_id,
1857 file,
1858 downloaded_bytes: downloaded.load(Ordering::Relaxed),
1859 total_bytes: total,
1860 });
1861 break;
1862 }
1863 }
1864 }
1865 })
1866}
1867
1868#[cfg(test)]
1869mod tests {
1870 use super::*;
1871
1872 fn validator(size: Option<u64>, etag: Option<&str>) -> Validator {
1873 Validator {
1874 etag: etag.map(str::to_string),
1875 last_modified: None,
1876 size,
1877 }
1878 }
1879
1880 #[test]
1881 fn mirror_compatibility_requires_matching_size() {
1882 assert!(mirror_compatible(
1883 &validator(Some(10), Some("\"a\"")),
1884 &validator(Some(10), Some("\"a\"")),
1885 false
1886 ));
1887 assert!(!mirror_compatible(
1888 &validator(Some(10), None),
1889 &validator(Some(11), None),
1890 true
1891 ));
1892 }
1893
1894 #[test]
1895 fn mirror_compatibility_uses_checksum_when_size_is_unknown() {
1896 assert!(!mirror_compatible(
1897 &validator(None, None),
1898 &validator(None, None),
1899 false
1900 ));
1901 assert!(mirror_compatible(
1902 &validator(None, None),
1903 &validator(None, None),
1904 true
1905 ));
1906 }
1907
1908 #[test]
1909 fn backoff_is_jittered_but_bounded() {
1910 let delay = backoff(0);
1911 assert!(delay >= Duration::from_millis(250));
1912 assert!(delay <= Duration::from_millis(312));
1913 }
1914
1915 #[test]
1916 fn mirror_compatibility_rejects_different_strong_etags() {
1917 assert!(!mirror_compatible(
1918 &validator(Some(10), Some("\"a\"")),
1919 &validator(Some(10), Some("\"b\"")),
1920 false
1921 ));
1922 assert!(mirror_compatible(
1923 &validator(Some(10), Some("\"a\"")),
1924 &validator(Some(10), Some("\"b\"")),
1925 true
1926 ));
1927 }
1928}