Skip to main content

shaul_http/
backend.rs

1use std::collections::{BTreeMap, HashMap, VecDeque};
2use std::path::{Path, PathBuf};
3use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
4use std::sync::{Arc, Mutex as StdMutex};
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use async_trait::async_trait;
8use base64::Engine;
9use futures_util::StreamExt;
10use reqwest::header::{
11    ACCEPT_ENCODING, AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, LOCATION,
12};
13use reqwest::redirect::Policy;
14use reqwest::{Client, Response, StatusCode};
15use shaul_core::{
16    BackendCapabilities, Checksum, ChecksumAlgorithm, Config, DownloadBackend, DownloadOutcome,
17    DownloadRequest, Error, EventKind, EventSink, FileOutcome, FileResume, Header, Resource,
18    ResourceFile, ResourceKind, Result, ResumeCheckpoint, TaskId, TaskResume, Validator,
19    VerifyMode, completed_bytes, merge_ranges, missing_ranges, part_path, sanitize_file_name,
20    split_range,
21};
22use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
23use tokio::sync::Mutex;
24use tokio_util::sync::CancellationToken;
25use url::Url;
26
27use crate::budget::{ConnectionBudget, ConnectionPermit};
28use crate::filename::parse_content_disposition;
29use crate::rate::{RateLimiter, RateLimiters};
30
31const MAX_REDIRECTS: usize = 10;
32
33const CAPTURE_HEADERS: &[&str] = &[
34    "content-disposition",
35    "etag",
36    "last-modified",
37    "content-length",
38    "accept-ranges",
39    "content-range",
40    "digest",
41    "content-md5",
42    "x-linked-etag",
43    "x-linked-size",
44    "x-repo-commit",
45    "x-xet-hash",
46    "x-xet-token-expiration",
47    "x-xet-refresh-route",
48    "x-error-code",
49];
50
51const RESERVED_HEADERS: &[&str] = &[
52    "range",
53    "if-range",
54    "host",
55    "content-length",
56    "accept-encoding",
57    "connection",
58    "transfer-encoding",
59];
60
61type TaskRates = Arc<StdMutex<HashMap<TaskId, (u64, Arc<RateLimiter>)>>>;
62
63/// Segmented, resumable HTTP/HTTPS download backend.
64#[derive(Clone)]
65pub struct HttpBackend {
66    client: Client,
67    budget: Arc<ConnectionBudget>,
68    global_rate: Option<Arc<RateLimiter>>,
69    task_rates: TaskRates,
70}
71
72impl HttpBackend {
73    /// Build a backend using the connection, proxy and user-agent settings
74    /// from `config`.
75    pub fn new(config: &Config) -> Result<Self> {
76        let mut builder = Client::builder()
77            .redirect(Policy::none())
78            .connect_timeout(Duration::from_secs(
79                config.network.connect_timeout_secs.max(1),
80            ))
81            .pool_max_idle_per_host(config.download.per_host.max(1))
82            .user_agent(config.network.user_agent.clone());
83        if let Some(proxy) = proxy_from_config(config)? {
84            builder = builder.proxy(proxy);
85        }
86        let client = builder
87            .build()
88            .map_err(|error| Error::Backend(format!("failed to build HTTP client: {error}")))?;
89        Ok(Self {
90            client,
91            budget: Arc::new(ConnectionBudget::new(
92                config.download.global,
93                config.download.per_host_active,
94            )),
95            global_rate: config
96                .download
97                .rate_limit
98                .filter(|value| *value > 0)
99                .map(|rate| Arc::new(RateLimiter::new(rate))),
100            task_rates: Arc::new(StdMutex::new(HashMap::new())),
101        })
102    }
103
104    fn request_rate(&self, request: &DownloadRequest) -> RateLimiters {
105        let task = request
106            .options
107            .task_rate_limit
108            .filter(|value| *value > 0)
109            .map(|rate| {
110                let mut rates = self
111                    .task_rates
112                    .lock()
113                    .unwrap_or_else(|poisoned| poisoned.into_inner());
114                if let Some((existing, limiter)) = rates.get(&request.task_id)
115                    && *existing == rate
116                {
117                    return limiter.clone();
118                }
119                let limiter = Arc::new(RateLimiter::new(rate));
120                rates.insert(request.task_id, (rate, limiter.clone()));
121                limiter
122            });
123        RateLimiters::new(self.global_rate.clone(), task)
124    }
125
126    /// Download one resource file, sharing the caller's task resume. This is
127    /// public so protocol-specific backends (Hub) can reuse the segmented,
128    /// resumable HTTP transfer without duplicating it.
129    pub async fn download_file(
130        &self,
131        request: &DownloadRequest,
132        file: &ResourceFile,
133        task_resume: &mut TaskResume,
134    ) -> Result<FileOutcome> {
135        let checksum = request
136            .options
137            .checksum
138            .clone()
139            .or_else(|| file.checksum.clone());
140        let mut reference: Option<Validator> = None;
141        let mut last_error: Option<Error> = None;
142        let mut urls = vec![file.url.clone()];
143        for mirror in &request.options.mirrors {
144            if !matches!(mirror.scheme(), "http" | "https") {
145                warning(request, format!("ignoring non-HTTP mirror URL: {mirror}"));
146                continue;
147            }
148            if !urls.contains(mirror) {
149                urls.push(mirror.clone());
150            }
151        }
152
153        for (index, url) in urls.iter().enumerate() {
154            let mut candidate = file.clone();
155            candidate.url = url.clone();
156            let probe = match self.probe(request, &candidate).await {
157                Ok(probe) => probe,
158                Err(error) if error.is_cancelled() => return Err(error),
159                Err(error) => {
160                    warning(request, format!("download URL {url} failed: {error}"));
161                    last_error = Some(error);
162                    continue;
163                }
164            };
165            let validator = probe.validator();
166            if let Some(reference) = &reference {
167                if !mirror_compatible(reference, &validator, checksum.is_some()) {
168                    warning(
169                        request,
170                        format!("mirror {url} is incompatible with the primary response"),
171                    );
172                    last_error = Some(Error::Backend(format!(
173                        "mirror {url} is incompatible with the primary response"
174                    )));
175                    continue;
176                }
177            } else {
178                reference = Some(validator.clone());
179            }
180            if index > 0 {
181                warning(request, format!("falling back to mirror {url}"));
182            }
183            match self
184                .download_file_with_probe(request, &candidate, task_resume, probe)
185                .await
186            {
187                Ok(outcome) => return Ok(outcome),
188                Err(error) if error.is_cancelled() => return Err(error),
189                Err(error) => {
190                    warning(request, format!("download URL {url} failed: {error}"));
191                    last_error = Some(error);
192                }
193            }
194        }
195        Err(last_error
196            .unwrap_or_else(|| Error::Backend("no HTTP download URL was available".to_string())))
197    }
198
199    #[tracing::instrument(
200        name = "shaul.http.file",
201        level = "debug",
202        skip(self, request, file, task_resume, probe),
203        fields(task_id = %request.task_id, file = %file.path)
204    )]
205    async fn download_file_with_probe(
206        &self,
207        request: &DownloadRequest,
208        file: &ResourceFile,
209        task_resume: &mut TaskResume,
210        probe: Probe,
211    ) -> Result<FileOutcome> {
212        let key = file.url.as_str().to_string();
213
214        if let Some(stored) = task_resume.files.get(&key).cloned()
215            && stored.is_complete()
216        {
217            let final_path = manifest_path(&request.destination, &stored.final_path);
218            let part = manifest_path(&request.destination, &stored.part_path);
219            if tokio::fs::try_exists(&final_path).await? {
220                let size = tokio::fs::metadata(&final_path).await?.len();
221                let relative = relative_path(&request.destination, &final_path);
222                request.events.emit(EventKind::FileStarted {
223                    task_id: request.task_id,
224                    file: relative.clone(),
225                    total_bytes: Some(size),
226                });
227                request.events.emit(EventKind::Progress {
228                    task_id: request.task_id,
229                    file: relative.clone(),
230                    downloaded_bytes: size,
231                    total_bytes: Some(size),
232                });
233                return Ok(FileOutcome::new(
234                    final_path,
235                    relative,
236                    size,
237                    stored.checksum.or(request.options.checksum.clone()),
238                    false,
239                ));
240            }
241            if tokio::fs::try_exists(&part).await? {
242                ensure_parent(&final_path).await?;
243                remove_if_exists(&final_path).await?;
244                tokio::fs::rename(&part, &final_path).await?;
245                let size = tokio::fs::metadata(&final_path).await?.len();
246                let relative = relative_path(&request.destination, &final_path);
247                request.events.emit(EventKind::FileStarted {
248                    task_id: request.task_id,
249                    file: relative.clone(),
250                    total_bytes: Some(size),
251                });
252                return Ok(FileOutcome::new(
253                    final_path,
254                    relative,
255                    size,
256                    stored.checksum.or(request.options.checksum.clone()),
257                    false,
258                ));
259            }
260        }
261
262        let total = probe.total_size;
263        let validator = probe.validator();
264        let checksum = request
265            .options
266            .checksum
267            .clone()
268            .or_else(|| probe.checksum.clone())
269            .or_else(|| file.checksum.clone());
270        if request.options.verify == VerifyMode::Require && checksum.is_none() {
271            return Err(Error::ChecksumMissing);
272        }
273
274        let default_name = file
275            .path
276            .rsplit('/')
277            .next()
278            .map(sanitize_file_name)
279            .unwrap_or_else(|| "download".to_string());
280        let (mut final_path, relative) =
281            resolve_final_path(request, file, probe.filename.as_deref(), &default_name)?;
282
283        let stored = task_resume.files.get(&key).cloned();
284        let final_for_manifest = path_for_manifest(&request.destination, &final_path);
285        let stored_matches_path = stored
286            .as_ref()
287            .map(|stored| stored.final_path == final_for_manifest)
288            .unwrap_or(false);
289        let resume_valid = request.options.resume
290            && stored
291                .as_ref()
292                .map(|stored| {
293                    stored_matches_path
294                        && !stored.validator.is_empty()
295                        && stored.validator.matches(&validator)
296                        && match (stored.total_size, total) {
297                            (Some(old), Some(new)) => old == new,
298                            _ => true,
299                        }
300                })
301                .unwrap_or(false);
302
303        if stored.is_some() && !resume_valid {
304            request.events.emit(EventKind::Warning {
305                task_id: Some(request.task_id),
306                message: "resume state is no longer valid; restarting this file".to_string(),
307            });
308        }
309
310        final_path = apply_conflict_policy(final_path, &request.options).await?;
311        let part_name = part_path(&final_path);
312        let part_for_manifest = path_for_manifest(&request.destination, &part_name);
313
314        let mut file_resume = FileResume {
315            url: key.clone(),
316            final_path: path_for_manifest(&request.destination, &final_path),
317            part_path: part_for_manifest.clone(),
318            validator: validator.clone(),
319            total_size: total,
320            segment_size: request.options.segment_size,
321            completed: if resume_valid {
322                stored
323                    .as_ref()
324                    .map(|stored| stored.completed.clone())
325                    .unwrap_or_default()
326            } else {
327                Vec::new()
328            },
329            checksum: checksum.clone(),
330            metadata: {
331                let mut metadata = probe.headers.clone();
332                metadata.insert("final-url".to_string(), redacted_url(&probe.final_url));
333                metadata
334            },
335            updated_at_ms: now_ms(),
336        };
337
338        // Reconcile existing partial data with the checkpoint.
339        let existing_part_len = tokio::fs::metadata(&part_name)
340            .await
341            .ok()
342            .map(|metadata| metadata.len());
343        if !resume_valid {
344            file_resume.completed.clear();
345        } else if let Some(len) = existing_part_len {
346            let limit = total.map(|total| total.min(len)).unwrap_or(len);
347            file_resume.completed = shaul_core::range::clamp_to_len(&file_resume.completed, limit);
348        } else {
349            file_resume.completed.clear();
350        }
351        file_resume.normalize();
352
353        ensure_parent(&final_path).await?;
354        if let Some(total) = total {
355            let file = tokio::fs::OpenOptions::new()
356                .create(true)
357                .write(true)
358                .truncate(false)
359                .open(&part_name)
360                .await?;
361            if !resume_valid {
362                file.set_len(0).await?;
363            }
364            file.set_len(total).await?;
365            file.sync_all().await?;
366        } else if !resume_valid {
367            tokio::fs::OpenOptions::new()
368                .create(true)
369                .write(true)
370                .truncate(true)
371                .open(&part_name)
372                .await?;
373        } else if !tokio::fs::try_exists(&part_name).await? {
374            tokio::fs::File::create(&part_name).await?;
375            file_resume.completed.clear();
376        }
377
378        task_resume.files.insert(key.clone(), file_resume.clone());
379        request.checkpoint.save(task_resume).await?;
380
381        let initial = completed_bytes(&file_resume.completed);
382        let progress = Arc::new(AtomicU64::new(initial));
383        request.events.emit(EventKind::FileStarted {
384            task_id: request.task_id,
385            file: relative.clone(),
386            total_bytes: total,
387        });
388
389        let progress_stop = CancellationToken::new();
390        let reporter = spawn_progress_reporter(
391            request.events.clone(),
392            request.task_id,
393            relative.clone(),
394            total,
395            progress.clone(),
396            progress_stop.clone(),
397        );
398
399        let rate = self.request_rate(request);
400        let transfer_result = if let Some(0) = total {
401            // Empty file: the pre-allocated part is already the final content.
402            task_resume.files.insert(key.clone(), file_resume.clone());
403            request.checkpoint.save(task_resume).await?;
404            Ok(file_resume.clone())
405        } else if probe.range_supported
406            && total.map(|total| total > 0).unwrap_or(false)
407            && request.options.connections > 1
408        {
409            self.transfer_segmented(
410                request,
411                file,
412                &part_name,
413                &key,
414                file_resume.clone(),
415                task_resume.clone(),
416                progress.clone(),
417                total,
418                &rate,
419            )
420            .await
421        } else {
422            self.transfer_single(
423                request,
424                file,
425                &part_name,
426                &key,
427                file_resume.clone(),
428                task_resume.clone(),
429                progress.clone(),
430                probe.range_supported,
431                &rate,
432            )
433            .await
434        };
435
436        progress_stop.cancel();
437        let _ = reporter.await;
438
439        let mut final_resume = match transfer_result {
440            Ok(resume) => resume,
441            Err(error) => return Err(error),
442        };
443
444        if let Some(total) = final_resume.total_size
445            && total > 0
446        {
447            final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
448        }
449        final_resume.normalize();
450        request.events.emit(EventKind::Progress {
451            task_id: request.task_id,
452            file: relative.clone(),
453            downloaded_bytes: final_resume.completed_bytes(),
454            total_bytes: final_resume.total_size,
455        });
456        task_resume.files.insert(key, final_resume.clone());
457        request.checkpoint.save(task_resume).await?;
458
459        ensure_parent(&final_path).await?;
460        remove_if_exists(&final_path).await?;
461        if tokio::fs::try_exists(&part_name).await? {
462            tokio::fs::rename(&part_name, &final_path).await?;
463        } else {
464            tokio::fs::write(&final_path, []).await?;
465        }
466        let size = tokio::fs::metadata(&final_path).await?.len();
467        Ok(FileOutcome::new(
468            final_path, relative, size, checksum, false,
469        ))
470    }
471
472    async fn probe(&self, request: &DownloadRequest, file: &ResourceFile) -> Result<Probe> {
473        let mut attempt = 0u32;
474        loop {
475            let headers = build_headers(&request.options, &file.headers, Some("bytes=0-0"), None)?;
476            let sent = self
477                .send_follow(&file.url, &headers, &request.options, &request.cancel)
478                .await;
479            match sent {
480                Err(Error::Cancelled) => return Err(Error::Cancelled),
481                Err(error) => {
482                    if attempt >= request.options.retries {
483                        return Err(error);
484                    }
485                    tracing::warn!(attempt, error = %error, "request failed; retrying");
486                    retry_event(request, file, attempt + 1, error.to_string());
487                    sleep_or_cancel(backoff(attempt), &request.cancel).await?;
488                    attempt += 1;
489                }
490                Ok((response, captured, final_url, _permit)) => {
491                    let status = response.status();
492                    let retry_after = retry_after(response.headers());
493                    if status.is_server_error()
494                        || status == StatusCode::REQUEST_TIMEOUT
495                        || status == StatusCode::TOO_MANY_REQUESTS
496                    {
497                        drop(response);
498                        if attempt >= request.options.retries {
499                            return Err(http_error(status, &captured));
500                        }
501                        tracing::warn!(
502                            attempt,
503                            status = status.as_u16(),
504                            "transient HTTP status; retrying"
505                        );
506                        retry_event(request, file, attempt + 1, format!("HTTP {status}"));
507                        sleep_or_cancel(
508                            retry_after.unwrap_or_else(|| backoff(attempt)),
509                            &request.cancel,
510                        )
511                        .await?;
512                        attempt += 1;
513                        continue;
514                    }
515                    if matches!(
516                        status,
517                        StatusCode::UNAUTHORIZED
518                            | StatusCode::FORBIDDEN
519                            | StatusCode::NOT_FOUND
520                            | StatusCode::GONE
521                    ) {
522                        return Err(http_error(status, &captured));
523                    }
524
525                    let content_range = captured
526                        .get("content-range")
527                        .and_then(|value| parse_content_range(value));
528                    let linked_size = captured
529                        .get("x-linked-size")
530                        .and_then(|value| parse_u64(value));
531                    let content_length = captured
532                        .get("content-length")
533                        .and_then(|value| parse_u64(value));
534                    let (range_supported, total_size) = match status {
535                        StatusCode::PARTIAL_CONTENT => (
536                            true,
537                            content_range
538                                .and_then(|(_, _, total)| total)
539                                .or(linked_size),
540                        ),
541                        StatusCode::OK => match content_range {
542                            // Some servers answer a ranged request with 200 +
543                            // Content-Range (ModelScope does this). The body is
544                            // still a partial range, so use the total from
545                            // Content-Range rather than the range length in
546                            // Content-Length.
547                            Some((_, _, total)) => (true, total.or(linked_size)),
548                            None => (false, linked_size.or(content_length)),
549                        },
550                        StatusCode::RANGE_NOT_SATISFIABLE => {
551                            (true, content_range.and_then(|(_, _, total)| total))
552                        }
553                        _ => {
554                            return Err(Error::Backend(format!(
555                                "unexpected HTTP status during probe: {status}"
556                            )));
557                        }
558                    };
559
560                    let checksum = captured
561                        .get("x-linked-etag")
562                        .and_then(|value| Checksum::from_etag(value))
563                        .or_else(|| parse_digest(&captured));
564                    let etag = captured
565                        .get("etag")
566                        .cloned()
567                        .or_else(|| captured.get("x-linked-etag").cloned());
568                    let last_modified = captured.get("last-modified").cloned();
569                    let filename = captured
570                        .get("content-disposition")
571                        .and_then(|value| parse_content_disposition(value));
572                    let mut headers = captured;
573                    if let Some(total) = total_size {
574                        headers.insert("total-size".to_string(), total.to_string());
575                    }
576                    return Ok(Probe {
577                        final_url,
578                        total_size,
579                        range_supported,
580                        filename,
581                        etag,
582                        last_modified,
583                        checksum,
584                        headers,
585                    });
586                }
587            }
588        }
589    }
590
591    async fn send_follow(
592        &self,
593        url: &Url,
594        headers: &HeaderMap,
595        options: &shaul_core::ResolvedOptions,
596        cancel: &CancellationToken,
597    ) -> Result<(Response, BTreeMap<String, String>, Url, ConnectionPermit)> {
598        let mut current = url.clone();
599        let mut headers = headers.clone();
600        let mut captured = BTreeMap::new();
601
602        for _ in 0..MAX_REDIRECTS {
603            let permit = self
604                .budget
605                .acquire(current.host_str().unwrap_or_default(), cancel)
606                .await?;
607            let response = tokio::select! {
608                _ = cancel.cancelled() => return Err(Error::Cancelled),
609                response = self.client.get(current.clone()).headers(headers.clone()).send() => {
610                    response.map_err(|error| Error::Backend(format!("request failed: {error}")))?
611                }
612            };
613            capture_headers(&response, &mut captured);
614            let status = response.status();
615            if matches!(
616                status,
617                StatusCode::MOVED_PERMANENTLY
618                    | StatusCode::FOUND
619                    | StatusCode::SEE_OTHER
620                    | StatusCode::TEMPORARY_REDIRECT
621                    | StatusCode::PERMANENT_REDIRECT
622            ) {
623                let location = response
624                    .headers()
625                    .get(LOCATION)
626                    .and_then(|value| value.to_str().ok())
627                    .ok_or_else(|| {
628                        Error::Backend("redirect response missing Location".to_string())
629                    })?;
630                let next = current
631                    .join(location)
632                    .map_err(|error| Error::Backend(format!("invalid redirect URL: {error}")))?;
633                if !same_origin(&current, &next) && !options.request.allow_cross_origin_auth {
634                    for name in options.request.sensitive_header_names() {
635                        headers.remove(name.as_str());
636                    }
637                    headers.remove(AUTHORIZATION);
638                    headers.remove(COOKIE);
639                }
640                current = next;
641                continue;
642            }
643            return Ok((response, captured, current, permit));
644        }
645        Err(Error::Backend("too many redirects".to_string()))
646    }
647
648    #[allow(clippy::too_many_arguments)]
649    async fn transfer_segmented(
650        &self,
651        request: &DownloadRequest,
652        file: &ResourceFile,
653        part_name: &Path,
654        key: &str,
655        file_resume: FileResume,
656        task_resume: TaskResume,
657        progress: Arc<AtomicU64>,
658        total: Option<u64>,
659        rate: &RateLimiters,
660    ) -> Result<FileResume> {
661        let total =
662            total.ok_or_else(|| Error::Backend("segmented transfer without size".to_string()))?;
663        let missing = missing_ranges(total, &file_resume.completed);
664        let mut chunks = Vec::new();
665        for range in missing {
666            chunks.extend(split_range(range, request.options.segment_size));
667        }
668        if chunks.is_empty() {
669            return Ok(file_resume);
670        }
671
672        let mut state = task_resume.clone();
673        state.files.insert(key.to_string(), file_resume.clone());
674        let resume_state = Arc::new(Mutex::new(state));
675        let checkpoint_lock = Arc::new(Mutex::new(()));
676        let queue = Arc::new(std::sync::Mutex::new(VecDeque::from(chunks)));
677        let range_broken = Arc::new(AtomicBool::new(false));
678        let child_cancel = request.cancel.child_token();
679        let if_range = strong_if_range(&file_resume.validator);
680        let workers = request
681            .options
682            .connections
683            .min(queue.lock().map(|queue| queue.len()).unwrap_or(1))
684            .max(1);
685
686        let mut set = tokio::task::JoinSet::new();
687        for _ in 0..workers {
688            let backend = self.clone();
689            let request = request.clone();
690            let file = file.clone();
691            let part_name = part_name.to_path_buf();
692            let key = key.to_string();
693            let queue = queue.clone();
694            let resume_state = resume_state.clone();
695            let checkpoint_lock = checkpoint_lock.clone();
696            let range_broken = range_broken.clone();
697            let progress = progress.clone();
698            let cancel = child_cancel.clone();
699            let if_range = if_range.clone();
700            let rate = rate.clone();
701            set.spawn(async move {
702                backend
703                    .segment_worker(
704                        &request,
705                        &file,
706                        &part_name,
707                        &key,
708                        queue,
709                        resume_state,
710                        checkpoint_lock,
711                        range_broken,
712                        progress,
713                        cancel,
714                        total,
715                        if_range,
716                        rate,
717                    )
718                    .await
719            });
720        }
721
722        let mut first_error = None;
723        while let Some(joined) = set.join_next().await {
724            match joined {
725                Ok(Ok(())) => {}
726                Ok(Err(FetchError::Cancelled)) => {}
727                Ok(Err(FetchError::RangeBroken)) => {}
728                Ok(Err(FetchError::Fatal(error))) => {
729                    if first_error.is_none() {
730                        first_error = Some(error);
731                    }
732                    child_cancel.cancel();
733                }
734                Ok(Err(FetchError::Retryable(message))) => {
735                    if first_error.is_none() {
736                        first_error = Some(Error::Backend(message));
737                    }
738                    child_cancel.cancel();
739                }
740                Err(error) => {
741                    if first_error.is_none() {
742                        first_error = Some(Error::Other(format!("segment worker failed: {error}")));
743                    }
744                    child_cancel.cancel();
745                }
746            }
747        }
748
749        if let Some(error) = first_error {
750            return Err(error);
751        }
752
753        if range_broken.load(Ordering::SeqCst) {
754            request.events.emit(EventKind::Warning {
755                task_id: Some(request.task_id),
756                message: "server ignored a ranged request; falling back to a single stream"
757                    .to_string(),
758            });
759            let file_handle = tokio::fs::OpenOptions::new()
760                .create(true)
761                .write(true)
762                .truncate(false)
763                .open(part_name)
764                .await?;
765            file_handle.set_len(0).await?;
766            file_handle.sync_all().await?;
767            progress.store(0, Ordering::Relaxed);
768            let mut reset = file_resume;
769            reset.completed.clear();
770            reset.normalize();
771            return self
772                .transfer_single(
773                    request,
774                    file,
775                    part_name,
776                    key,
777                    reset,
778                    task_resume,
779                    progress,
780                    false,
781                    rate,
782                )
783                .await;
784        }
785
786        let final_state = resume_state.lock().await.clone();
787        let mut final_resume = final_state
788            .files
789            .get(key)
790            .cloned()
791            .ok_or_else(|| Error::Backend("missing resume state after transfer".to_string()))?;
792        final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
793        final_resume.normalize();
794        let mut snapshot = final_state;
795        snapshot.files.insert(key.to_string(), final_resume.clone());
796        request.checkpoint.save(&snapshot).await?;
797        Ok(final_resume)
798    }
799
800    #[allow(clippy::too_many_arguments)]
801    async fn segment_worker(
802        &self,
803        request: &DownloadRequest,
804        file: &ResourceFile,
805        part_name: &Path,
806        key: &str,
807        queue: Arc<std::sync::Mutex<VecDeque<shaul_core::ByteRange>>>,
808        resume_state: Arc<Mutex<TaskResume>>,
809        checkpoint_lock: Arc<Mutex<()>>,
810        range_broken: Arc<AtomicBool>,
811        progress: Arc<AtomicU64>,
812        cancel: CancellationToken,
813        total: u64,
814        if_range: Option<String>,
815        rate: RateLimiters,
816    ) -> std::result::Result<(), FetchError> {
817        let mut part = tokio::fs::OpenOptions::new()
818            .write(true)
819            .open(part_name)
820            .await
821            .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
822        loop {
823            if cancel.is_cancelled() || range_broken.load(Ordering::SeqCst) {
824                break;
825            }
826            let range = {
827                let mut queue = queue
828                    .lock()
829                    .map_err(|_| FetchError::Fatal(Error::Other("work queue poisoned".into())))?;
830                queue.pop_front()
831            };
832            let Some(range) = range else {
833                break;
834            };
835            match self
836                .fetch_range(
837                    request,
838                    file,
839                    range,
840                    &mut part,
841                    &progress,
842                    &cancel,
843                    total,
844                    if_range.as_deref(),
845                    &rate,
846                )
847                .await
848            {
849                Ok(()) => {
850                    let _guard = checkpoint_lock.lock().await;
851                    let snapshot = {
852                        let mut state = resume_state.lock().await;
853                        if let Some(entry) = state.files.get_mut(key) {
854                            entry.completed.push(range);
855                            entry.normalize();
856                        }
857                        state.clone()
858                    };
859                    request
860                        .checkpoint
861                        .save(&snapshot)
862                        .await
863                        .map_err(FetchError::Fatal)?;
864                }
865                Err(FetchError::Cancelled) => break,
866                Err(FetchError::RangeBroken) => {
867                    range_broken.store(true, Ordering::SeqCst);
868                    cancel.cancel();
869                    break;
870                }
871                Err(error) => {
872                    cancel.cancel();
873                    return Err(error);
874                }
875            }
876        }
877        part.flush()
878            .await
879            .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
880        Ok(())
881    }
882
883    #[allow(clippy::too_many_arguments)]
884    async fn fetch_range(
885        &self,
886        request: &DownloadRequest,
887        file: &ResourceFile,
888        range: shaul_core::ByteRange,
889        part: &mut tokio::fs::File,
890        progress: &AtomicU64,
891        cancel: &CancellationToken,
892        total: u64,
893        if_range: Option<&str>,
894        rate: &RateLimiters,
895    ) -> std::result::Result<(), FetchError> {
896        let mut attempt = 0u32;
897        loop {
898            if cancel.is_cancelled() {
899                return Err(FetchError::Cancelled);
900            }
901            let range_header = format!("bytes={}-{}", range.start, range.end - 1);
902            let headers = build_headers(
903                &request.options,
904                &file.headers,
905                Some(&range_header),
906                if_range,
907            )
908            .map_err(FetchError::Fatal)?;
909            let sent = self
910                .send_follow(&file.url, &headers, &request.options, cancel)
911                .await;
912            let (response, captured, _, _permit) = match sent {
913                Err(Error::Cancelled) => return Err(FetchError::Cancelled),
914                Err(error) => {
915                    if attempt >= request.options.retries {
916                        return Err(FetchError::Fatal(error));
917                    }
918                    tracing::warn!(attempt, error = %error, "transfer request failed; retrying");
919                    retry_event(request, file, attempt + 1, error.to_string());
920                    sleep_or_cancel(backoff(attempt), cancel)
921                        .await
922                        .map_err(FetchError::Fatal)?;
923                    attempt += 1;
924                    continue;
925                }
926                Ok(response) => response,
927            };
928            let status = response.status();
929            let retry_after = retry_after(response.headers());
930
931            if status == StatusCode::RANGE_NOT_SATISFIABLE {
932                return Err(FetchError::Fatal(Error::Backend(format!(
933                    "range {} is not satisfiable",
934                    range_header
935                ))));
936            }
937            if status.is_server_error()
938                || status == StatusCode::REQUEST_TIMEOUT
939                || status == StatusCode::TOO_MANY_REQUESTS
940            {
941                drop(response);
942                if attempt >= request.options.retries {
943                    return Err(FetchError::Fatal(http_error(status, &captured)));
944                }
945                tracing::warn!(
946                    attempt,
947                    status = status.as_u16(),
948                    "transient HTTP status during transfer; retrying"
949                );
950                retry_event(request, file, attempt + 1, format!("HTTP {status}"));
951                sleep_or_cancel(retry_after.unwrap_or_else(|| backoff(attempt)), cancel)
952                    .await
953                    .map_err(FetchError::Fatal)?;
954                attempt += 1;
955                continue;
956            }
957
958            let content_range = captured
959                .get("content-range")
960                .and_then(|value| parse_content_range(value));
961            if status == StatusCode::OK && content_range.is_none() {
962                return Err(FetchError::RangeBroken);
963            }
964            let is_partial = status == StatusCode::PARTIAL_CONTENT
965                || (status == StatusCode::OK && content_range.is_some());
966            if !is_partial {
967                return Err(FetchError::Fatal(http_error(status, &captured)));
968            }
969            let parsed = content_range.ok_or_else(|| {
970                FetchError::Fatal(Error::Backend(
971                    "ranged response missing a valid Content-Range".to_string(),
972                ))
973            })?;
974            if parsed.0 != range.start || parsed.1 + 1 != range.end {
975                return Err(FetchError::Fatal(Error::Backend(format!(
976                    "server returned range {}-{}, expected {}-{}",
977                    parsed.0,
978                    parsed.1,
979                    range.start,
980                    range.end - 1
981                ))));
982            }
983            if let Some(server_total) = parsed.2
984                && server_total != total
985            {
986                return Err(FetchError::Fatal(Error::Backend(format!(
987                    "file size changed from {total} to {server_total}"
988                ))));
989            }
990
991            part.seek(SeekFrom::Start(range.start))
992                .await
993                .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
994            let mut written = 0u64;
995            let mut stream = response.bytes_stream();
996            let retry_message;
997            loop {
998                let next = tokio::select! {
999                    _ = cancel.cancelled() => return Err(FetchError::Cancelled),
1000                    next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1001                };
1002                match next {
1003                    Ok(Some(Ok(chunk))) => {
1004                        if written + chunk.len() as u64 > range.len() {
1005                            retry_message =
1006                                Some("server sent more data than requested".to_string());
1007                            break;
1008                        }
1009                        rate.throttle(chunk.len() as u64, cancel)
1010                            .await
1011                            .map_err(FetchError::Fatal)?;
1012                        part.write_all(&chunk)
1013                            .await
1014                            .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
1015                        written += chunk.len() as u64;
1016                        progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1017                    }
1018                    Ok(Some(Err(error))) => {
1019                        retry_message = Some(format!("stream error: {error}"));
1020                        break;
1021                    }
1022                    Ok(None) => {
1023                        if written == range.len() {
1024                            return Ok(());
1025                        }
1026                        retry_message = Some(format!(
1027                            "short ranged response: {written} of {} bytes",
1028                            range.len()
1029                        ));
1030                        break;
1031                    }
1032                    Err(_) => {
1033                        retry_message = Some("read timed out".to_string());
1034                        break;
1035                    }
1036                }
1037            }
1038
1039            // A retry rewrites this range from the beginning; remove the
1040            // partial bytes from the progress counter first.
1041            let current = progress.load(Ordering::Relaxed);
1042            progress.fetch_sub(written.min(current), Ordering::Relaxed);
1043            let message = retry_message.unwrap_or_else(|| "range transfer failed".to_string());
1044            if attempt >= request.options.retries {
1045                return Err(FetchError::Retryable(message));
1046            }
1047            retry_event(request, file, attempt + 1, message);
1048            sleep_or_cancel(backoff(attempt), cancel)
1049                .await
1050                .map_err(FetchError::Fatal)?;
1051            attempt += 1;
1052        }
1053    }
1054
1055    #[allow(clippy::too_many_arguments)]
1056    async fn transfer_single(
1057        &self,
1058        request: &DownloadRequest,
1059        file: &ResourceFile,
1060        part_name: &Path,
1061        key: &str,
1062        mut file_resume: FileResume,
1063        mut task_resume: TaskResume,
1064        progress: Arc<AtomicU64>,
1065        range_supported: bool,
1066        rate: &RateLimiters,
1067    ) -> Result<FileResume> {
1068        let mut offset = if request.options.resume && range_supported {
1069            contiguous_end(&file_resume.completed)
1070        } else {
1071            0
1072        };
1073        if file_resume
1074            .total_size
1075            .map(|total| offset >= total)
1076            .unwrap_or(false)
1077        {
1078            return Ok(file_resume);
1079        }
1080
1081        let mut part = tokio::fs::OpenOptions::new()
1082            .create(true)
1083            .write(true)
1084            .truncate(false)
1085            .open(part_name)
1086            .await?;
1087        if offset == 0 {
1088            part.set_len(0).await?;
1089        }
1090        part.seek(SeekFrom::Start(offset)).await?;
1091
1092        let mut attempt = 0u32;
1093        loop {
1094            if request.cancel.is_cancelled() {
1095                checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1096                return Err(Error::Cancelled);
1097            }
1098            let range = if offset > 0 && range_supported {
1099                Some(format!("bytes={offset}-"))
1100            } else {
1101                None
1102            };
1103            let if_range = if offset > 0 {
1104                strong_if_range(&file_resume.validator)
1105            } else {
1106                None
1107            };
1108            let headers = build_headers(
1109                &request.options,
1110                &file.headers,
1111                range.as_deref(),
1112                if_range.as_deref(),
1113            )?;
1114            let sent = self
1115                .send_follow(&file.url, &headers, &request.options, &request.cancel)
1116                .await;
1117            let (response, captured, _, _permit) = match sent {
1118                Err(Error::Cancelled) => {
1119                    checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
1120                        .await?;
1121                    return Err(Error::Cancelled);
1122                }
1123                Err(error) => {
1124                    checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
1125                        .await?;
1126                    if attempt >= request.options.retries {
1127                        return Err(error);
1128                    }
1129                    retry_event(request, file, attempt + 1, error.to_string());
1130                    sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1131                    attempt += 1;
1132                    continue;
1133                }
1134                Ok(response) => response,
1135            };
1136            let status = response.status();
1137            let retry_after = retry_after(response.headers());
1138
1139            let content_range = captured
1140                .get("content-range")
1141                .and_then(|value| parse_content_range(value));
1142            if status.is_server_error()
1143                || status == StatusCode::REQUEST_TIMEOUT
1144                || status == StatusCode::TOO_MANY_REQUESTS
1145            {
1146                drop(response);
1147                checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1148                if attempt >= request.options.retries {
1149                    return Err(http_error(status, &captured));
1150                }
1151                retry_event(request, file, attempt + 1, format!("HTTP {status}"));
1152                sleep_or_cancel(
1153                    retry_after.unwrap_or_else(|| backoff(attempt)),
1154                    &request.cancel,
1155                )
1156                .await?;
1157                attempt += 1;
1158                continue;
1159            }
1160
1161            let is_partial = status == StatusCode::PARTIAL_CONTENT
1162                || (status == StatusCode::OK && content_range.is_some());
1163            if is_partial {
1164                if let Some((start, _, _)) = content_range
1165                    && offset > 0
1166                    && start != offset
1167                {
1168                    return Err(Error::Backend(format!(
1169                        "server resumed at {start}, expected {offset}"
1170                    )));
1171                }
1172            } else if status == StatusCode::OK {
1173                if offset > 0 {
1174                    part.set_len(0).await?;
1175                    part.seek(SeekFrom::Start(0)).await?;
1176                    offset = 0;
1177                    progress.store(0, Ordering::Relaxed);
1178                }
1179            } else {
1180                return Err(http_error(status, &captured));
1181            }
1182
1183            part.seek(SeekFrom::Start(offset)).await?;
1184            let mut stream = response.bytes_stream();
1185            let mut since_checkpoint = 0u64;
1186            let retry_message;
1187            loop {
1188                let next = tokio::select! {
1189                    _ = request.cancel.cancelled() => {
1190                        checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1191                        part.sync_all().await?;
1192                        return Err(Error::Cancelled);
1193                    }
1194                    next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1195                };
1196                match next {
1197                    Ok(Some(Ok(chunk))) => {
1198                        if let Some(total) = file_resume.total_size
1199                            && offset + chunk.len() as u64 > total
1200                        {
1201                            return Err(Error::Backend(
1202                                "server sent more data than the known file size".to_string(),
1203                            ));
1204                        }
1205                        rate.throttle(chunk.len() as u64, &request.cancel).await?;
1206                        part.write_all(&chunk).await?;
1207                        offset += chunk.len() as u64;
1208                        progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1209                        since_checkpoint += chunk.len() as u64;
1210                        if since_checkpoint >= request.options.segment_size {
1211                            checkpoint_single(
1212                                request,
1213                                &mut task_resume,
1214                                key,
1215                                &mut file_resume,
1216                                offset,
1217                            )
1218                            .await?;
1219                            since_checkpoint = 0;
1220                        }
1221                    }
1222                    Ok(Some(Err(error))) => {
1223                        retry_message = Some(format!("stream error: {error}"));
1224                        break;
1225                    }
1226                    Ok(None) => {
1227                        if let Some(total) = file_resume.total_size
1228                            && offset < total
1229                        {
1230                            retry_message =
1231                                Some(format!("short response: {offset} of {total} bytes"));
1232                            break;
1233                        }
1234                        part.flush().await?;
1235                        part.sync_all().await?;
1236                        file_resume.total_size = file_resume.total_size.or(Some(offset));
1237                        file_resume.completed = if offset > 0 {
1238                            vec![shaul_core::ByteRange::new(0, offset)?]
1239                        } else {
1240                            Vec::new()
1241                        };
1242                        file_resume.normalize();
1243                        task_resume
1244                            .files
1245                            .insert(key.to_string(), file_resume.clone());
1246                        request.checkpoint.save(&task_resume).await?;
1247                        return Ok(file_resume);
1248                    }
1249                    Err(_) => {
1250                        retry_message = Some("read timed out".to_string());
1251                        break;
1252                    }
1253                }
1254            }
1255
1256            part.flush().await?;
1257            checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1258            let message = retry_message.unwrap_or_else(|| "download failed".to_string());
1259            if attempt >= request.options.retries {
1260                return Err(Error::Backend(message));
1261            }
1262            retry_event(request, file, attempt + 1, message);
1263            sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1264            attempt += 1;
1265        }
1266    }
1267}
1268
1269#[async_trait]
1270impl DownloadBackend for HttpBackend {
1271    fn name(&self) -> &'static str {
1272        "http"
1273    }
1274
1275    fn capabilities(&self) -> BackendCapabilities {
1276        BackendCapabilities::new(true, true, true)
1277    }
1278
1279    fn supports(&self, resource: &Resource) -> bool {
1280        matches!(&resource.kind, ResourceKind::Http)
1281    }
1282
1283    #[tracing::instrument(
1284        name = "shaul.http.download",
1285        level = "debug",
1286        skip(self, request),
1287        fields(task_id = %request.task_id, files = request.resource.files.len())
1288    )]
1289    async fn download(&self, request: DownloadRequest) -> Result<DownloadOutcome> {
1290        if request.options.file_jobs > 1 && request.resource.files.len() > 1 {
1291            self.download_concurrent(&request).await
1292        } else {
1293            self.download_sequential(&request).await
1294        }
1295    }
1296}
1297
1298impl HttpBackend {
1299    async fn download_sequential(&self, request: &DownloadRequest) -> Result<DownloadOutcome> {
1300        let mut task_resume = request.resume.clone().unwrap_or_default();
1301        let mut outcomes = Vec::new();
1302        for file in &request.resource.files {
1303            if request.cancel.is_cancelled() {
1304                return Err(Error::Cancelled);
1305            }
1306            let outcome = self.download_file(request, file, &mut task_resume).await?;
1307            outcomes.push(outcome);
1308            request.checkpoint.save(&task_resume).await?;
1309        }
1310        Ok(DownloadOutcome::Completed(outcomes))
1311    }
1312
1313    async fn download_concurrent(&self, request: &DownloadRequest) -> Result<DownloadOutcome> {
1314        let shared = Arc::new(Mutex::new(request.resume.clone().unwrap_or_default()));
1315        let cancel = request.cancel.child_token();
1316        let semaphore = Arc::new(tokio::sync::Semaphore::new(
1317            request.options.file_jobs.max(1),
1318        ));
1319        let mut tasks = tokio::task::JoinSet::new();
1320        for (index, file) in request.resource.files.iter().cloned().enumerate() {
1321            let key = file.url.as_str().to_string();
1322            let mut local = TaskResume::default();
1323            if let Some(stored) = shared.lock().await.files.get(&key).cloned() {
1324                local.files.insert(key, stored);
1325            }
1326            let mut file_request = request.clone();
1327            file_request.cancel = cancel.clone();
1328            let checkpoint = Arc::new(FileCheckpoint {
1329                shared: shared.clone(),
1330                inner: request.checkpoint.clone(),
1331            });
1332            file_request.checkpoint = checkpoint.clone();
1333            let backend = self.clone();
1334            let permit = semaphore.clone();
1335            tasks.spawn(async move {
1336                let _permit = permit
1337                    .acquire_owned()
1338                    .await
1339                    .map_err(|_| Error::Other("file job semaphore closed".to_string()))?;
1340                let mut local = local;
1341                let outcome = backend
1342                    .download_file(&file_request, &file, &mut local)
1343                    .await?;
1344                checkpoint.save(&local).await?;
1345                Ok::<_, Error>((index, outcome))
1346            });
1347        }
1348
1349        let mut outcomes: Vec<Option<FileOutcome>> =
1350            (0..request.resource.files.len()).map(|_| None).collect();
1351        let mut first_error = None;
1352        while let Some(joined) = tasks.join_next().await {
1353            match joined {
1354                Ok(Ok((index, outcome))) => outcomes[index] = Some(outcome),
1355                Ok(Err(error)) => {
1356                    first_error.get_or_insert(error);
1357                    cancel.cancel();
1358                }
1359                Err(error) => {
1360                    first_error
1361                        .get_or_insert_with(|| Error::Other(format!("file job failed: {error}")));
1362                    cancel.cancel();
1363                }
1364            }
1365        }
1366        if let Some(error) = first_error {
1367            return Err(error);
1368        }
1369        let outcomes = outcomes
1370            .into_iter()
1371            .map(|outcome| outcome.ok_or_else(|| Error::Other("missing file outcome".to_string())))
1372            .collect::<Result<Vec<_>>>()?;
1373        Ok(DownloadOutcome::Completed(outcomes))
1374    }
1375}
1376
1377struct FileCheckpoint {
1378    shared: Arc<Mutex<TaskResume>>,
1379    inner: Arc<dyn ResumeCheckpoint>,
1380}
1381
1382#[async_trait]
1383impl ResumeCheckpoint for FileCheckpoint {
1384    async fn save(&self, resume: &TaskResume) -> Result<()> {
1385        let mut shared = self.shared.lock().await;
1386        for (key, file) in &resume.files {
1387            shared.files.insert(key.clone(), file.clone());
1388        }
1389        self.inner.save(&shared).await
1390    }
1391}
1392
1393fn warning(request: &DownloadRequest, message: impl Into<String>) {
1394    request.events.emit(EventKind::Warning {
1395        task_id: Some(request.task_id),
1396        message: message.into(),
1397    });
1398}
1399
1400fn retry_event(
1401    request: &DownloadRequest,
1402    file: &ResourceFile,
1403    attempt: u32,
1404    error: impl Into<String>,
1405) {
1406    request.events.emit(EventKind::Retry {
1407        task_id: request.task_id,
1408        file: file.path.clone(),
1409        attempt,
1410        error: error.into(),
1411    });
1412}
1413
1414fn mirror_compatible(reference: &Validator, mirror: &Validator, checksum_known: bool) -> bool {
1415    match (reference.size, mirror.size) {
1416        (Some(reference), Some(mirror)) if reference != mirror => return false,
1417        (None, _) | (_, None) if !checksum_known => return false,
1418        _ => {}
1419    }
1420    if !checksum_known
1421        && let (Some(reference), Some(mirror)) = (&reference.etag, &mirror.etag)
1422        && !reference.starts_with("W/")
1423        && !mirror.starts_with("W/")
1424        && reference != mirror
1425    {
1426        return false;
1427    }
1428    true
1429}
1430
1431struct Probe {
1432    final_url: Url,
1433    total_size: Option<u64>,
1434    range_supported: bool,
1435    filename: Option<String>,
1436    etag: Option<String>,
1437    last_modified: Option<String>,
1438    checksum: Option<Checksum>,
1439    headers: BTreeMap<String, String>,
1440}
1441
1442impl Probe {
1443    fn validator(&self) -> Validator {
1444        Validator {
1445            etag: self.etag.clone(),
1446            last_modified: self.last_modified.clone(),
1447            size: self.total_size,
1448        }
1449    }
1450}
1451
1452#[derive(Debug)]
1453enum FetchError {
1454    RangeBroken,
1455    Cancelled,
1456    Retryable(String),
1457    Fatal(Error),
1458}
1459
1460fn build_headers(
1461    options: &shaul_core::ResolvedOptions,
1462    file_headers: &[Header],
1463    range: Option<&str>,
1464    if_range: Option<&str>,
1465) -> Result<HeaderMap> {
1466    let mut headers = HeaderMap::new();
1467    for header in &options.request.headers {
1468        if !is_reserved(&header.name) {
1469            insert_header(
1470                &mut headers,
1471                &header.name,
1472                &shaul_core::expand_env(&header.value)?,
1473            )?;
1474        }
1475    }
1476    for header in file_headers {
1477        if !is_reserved(&header.name) {
1478            insert_header(
1479                &mut headers,
1480                &header.name,
1481                &shaul_core::expand_env(&header.value)?,
1482            )?;
1483        }
1484    }
1485
1486    if !headers.contains_key(AUTHORIZATION)
1487        && let Some(auth) = &options.request.auth
1488    {
1489        match auth {
1490            shaul_core::Auth::Bearer { token } => {
1491                insert_header(
1492                    &mut headers,
1493                    "authorization",
1494                    &format!("Bearer {}", shaul_core::expand_env(token)?),
1495                )?;
1496            }
1497            shaul_core::Auth::Basic { username, password } => {
1498                let encoded = base64::engine::general_purpose::STANDARD.encode(format!(
1499                    "{}:{}",
1500                    shaul_core::expand_env(username)?,
1501                    shaul_core::expand_env(password)?
1502                ));
1503                insert_header(&mut headers, "authorization", &format!("Basic {encoded}"))?;
1504            }
1505            shaul_core::Auth::Header(header) => {
1506                insert_header(
1507                    &mut headers,
1508                    &header.name,
1509                    &shaul_core::expand_env(&header.value)?,
1510                )?;
1511            }
1512        }
1513    }
1514
1515    headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
1516    insert_header(&mut headers, "user-agent", &options.user_agent)?;
1517    if let Some(range) = range {
1518        insert_header(&mut headers, "range", range)?;
1519    }
1520    if let Some(if_range) = if_range {
1521        insert_header(&mut headers, "if-range", if_range)?;
1522    }
1523    Ok(headers)
1524}
1525
1526fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<()> {
1527    let name = HeaderName::from_bytes(name.as_bytes())
1528        .map_err(|_| Error::Config(format!("invalid header name: {name}")))?;
1529    let value = HeaderValue::from_str(value)
1530        .map_err(|_| Error::Config(format!("invalid value for header: {name}")))?;
1531    headers.insert(name, value);
1532    Ok(())
1533}
1534
1535fn is_reserved(name: &str) -> bool {
1536    RESERVED_HEADERS
1537        .iter()
1538        .any(|reserved| name.eq_ignore_ascii_case(reserved))
1539}
1540
1541fn capture_headers(response: &Response, captured: &mut BTreeMap<String, String>) {
1542    for name in CAPTURE_HEADERS {
1543        if let Some(value) = response.headers().get(*name)
1544            && let Ok(value) = value.to_str()
1545        {
1546            captured.insert((*name).to_string(), value.to_string());
1547        }
1548    }
1549}
1550
1551fn redacted_url(url: &Url) -> String {
1552    let mut url = url.clone();
1553    url.set_query(None);
1554    url.set_fragment(None);
1555    url.to_string()
1556}
1557
1558fn same_origin(left: &Url, right: &Url) -> bool {
1559    left.scheme() == right.scheme()
1560        && left.host_str() == right.host_str()
1561        && left.port_or_known_default() == right.port_or_known_default()
1562}
1563
1564fn parse_content_range(value: &str) -> Option<(u64, u64, Option<u64>)> {
1565    let rest = value.trim().strip_prefix("bytes")?.trim();
1566    let (range, total) = rest.split_once('/')?;
1567    let total = if total.trim() == "*" {
1568        None
1569    } else {
1570        Some(total.trim().parse::<u64>().ok()?)
1571    };
1572    if range.trim() == "*" {
1573        return Some((0, 0, total));
1574    }
1575    let (start, end) = range.split_once('-')?;
1576    Some((
1577        start.trim().parse::<u64>().ok()?,
1578        end.trim().parse::<u64>().ok()?,
1579        total,
1580    ))
1581}
1582
1583fn parse_u64(value: &str) -> Option<u64> {
1584    value.trim().parse::<u64>().ok()
1585}
1586
1587fn retry_after(headers: &HeaderMap) -> Option<Duration> {
1588    headers
1589        .get("retry-after")
1590        .and_then(|value| value.to_str().ok())
1591        .and_then(|value| value.trim().parse::<u64>().ok())
1592        .map(|seconds| Duration::from_secs(seconds.min(60)))
1593}
1594
1595fn backoff(attempt: u32) -> Duration {
1596    let base = 250u64.saturating_mul(1u64 << attempt.min(5));
1597    let jitter = SystemTime::now()
1598        .duration_since(UNIX_EPOCH)
1599        .map(|elapsed| u64::from(elapsed.subsec_nanos()) % (base / 4 + 1))
1600        .unwrap_or(0);
1601    Duration::from_millis(base.saturating_add(jitter))
1602}
1603
1604async fn sleep_or_cancel(duration: Duration, cancel: &CancellationToken) -> Result<()> {
1605    tokio::select! {
1606        _ = cancel.cancelled() => Err(Error::Cancelled),
1607        _ = tokio::time::sleep(duration) => Ok(()),
1608    }
1609}
1610
1611fn proxy_from_config(config: &Config) -> Result<Option<reqwest::Proxy>> {
1612    let Some(url) = config.network.proxy.as_deref() else {
1613        return Ok(None);
1614    };
1615    let mut proxy = reqwest::Proxy::all(url)
1616        .map_err(|error| Error::Config(format!("invalid proxy URL: {error}")))?;
1617    if let Some(value) = config.network.no_proxy.as_deref()
1618        && let Some(no_proxy) = reqwest::NoProxy::from_string(value)
1619    {
1620        proxy = proxy.no_proxy(Some(no_proxy));
1621    }
1622    Ok(Some(proxy))
1623}
1624
1625fn http_error(status: StatusCode, captured: &BTreeMap<String, String>) -> Error {
1626    let code = captured
1627        .get("x-error-code")
1628        .map(|code| format!(" ({code})"))
1629        .unwrap_or_default();
1630    Error::Backend(format!("HTTP {status}{code}"))
1631}
1632
1633fn parse_digest(captured: &BTreeMap<String, String>) -> Option<Checksum> {
1634    if let Some(value) = captured.get("digest") {
1635        for part in value.split(',') {
1636            let Some((algorithm, encoded)) = part.trim().split_once('=') else {
1637                continue;
1638            };
1639            let algorithm = match algorithm.trim().to_ascii_lowercase().as_str() {
1640                "sha-256" | "sha256" => ChecksumAlgorithm::Sha256,
1641                "sha-512" | "sha512" => ChecksumAlgorithm::Sha512,
1642                "sha" | "sha-1" | "sha1" => ChecksumAlgorithm::Sha1,
1643                "md5" => ChecksumAlgorithm::Md5,
1644                _ => continue,
1645            };
1646            if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(encoded.trim())
1647                && let Ok(checksum) = Checksum::new(algorithm, hex::encode(bytes))
1648            {
1649                return Some(checksum);
1650            }
1651        }
1652    }
1653    if let Some(value) = captured.get("content-md5")
1654        && let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(value.trim())
1655        && let Ok(checksum) = Checksum::new(ChecksumAlgorithm::Md5, hex::encode(bytes))
1656    {
1657        return Some(checksum);
1658    }
1659    None
1660}
1661
1662fn strong_if_range(validator: &Validator) -> Option<String> {
1663    if let Some(etag) = &validator.etag
1664        && !etag.starts_with("W/")
1665    {
1666        return Some(etag.clone());
1667    }
1668    validator.last_modified.clone()
1669}
1670
1671fn contiguous_end(completed: &[shaul_core::ByteRange]) -> u64 {
1672    let merged = merge_ranges(completed.to_vec());
1673    if merged.len() == 1 && merged[0].start == 0 {
1674        merged[0].end
1675    } else {
1676        0
1677    }
1678}
1679
1680fn manifest_path(destination: &Path, value: &str) -> PathBuf {
1681    let path = Path::new(value);
1682    if path.is_absolute() {
1683        path.to_path_buf()
1684    } else {
1685        destination.join(path)
1686    }
1687}
1688
1689fn path_for_manifest(destination: &Path, path: &Path) -> String {
1690    match path.strip_prefix(destination) {
1691        Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1692        Err(_) => path.to_string_lossy().to_string(),
1693    }
1694}
1695
1696fn relative_path(destination: &Path, path: &Path) -> String {
1697    match path.strip_prefix(destination) {
1698        Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1699        Err(_) => path
1700            .file_name()
1701            .map(|name| name.to_string_lossy().to_string())
1702            .unwrap_or_else(|| "download".to_string()),
1703    }
1704}
1705
1706fn resolve_final_path(
1707    request: &DownloadRequest,
1708    file: &ResourceFile,
1709    probe_filename: Option<&str>,
1710    default_name: &str,
1711) -> Result<(PathBuf, String)> {
1712    let single = request.resource.files.len() == 1;
1713    let response_name = if file.use_response_filename {
1714        probe_filename.map(sanitize_file_name)
1715    } else {
1716        None
1717    };
1718    let name = response_name
1719        .clone()
1720        .unwrap_or_else(|| default_name.to_string());
1721
1722    if let Some(output) = &request.options.output {
1723        if single {
1724            if output.is_dir() {
1725                let relative = if file.use_response_filename {
1726                    name.clone()
1727                } else {
1728                    file.path.clone()
1729                };
1730                let path = output.join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1731                return Ok((path.clone(), relative_path(&request.destination, &path)));
1732            }
1733            return Ok((output.clone(), relative_path(&request.destination, output)));
1734        }
1735        let path = output.join(file.path.replace('/', std::path::MAIN_SEPARATOR_STR));
1736        return Ok((path.clone(), relative_path(&request.destination, &path)));
1737    }
1738
1739    let relative = if single && file.use_response_filename {
1740        name
1741    } else {
1742        file.path.clone()
1743    };
1744    let path = request
1745        .destination
1746        .join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1747    Ok((path.clone(), relative_path(&request.destination, &path)))
1748}
1749
1750async fn apply_conflict_policy(
1751    path: PathBuf,
1752    options: &shaul_core::ResolvedOptions,
1753) -> Result<PathBuf> {
1754    if !tokio::fs::try_exists(&path).await? {
1755        return Ok(path);
1756    }
1757    if options.overwrite {
1758        remove_if_exists(&path).await?;
1759        return Ok(path);
1760    }
1761    if options.rename_on_conflict {
1762        return unique_path(&path).await;
1763    }
1764    Err(Error::Backend(format!(
1765        "destination already exists: {} (use --force or --rename)",
1766        path.display()
1767    )))
1768}
1769
1770async fn unique_path(path: &Path) -> Result<PathBuf> {
1771    let stem = path
1772        .file_stem()
1773        .map(|stem| stem.to_string_lossy().to_string())
1774        .unwrap_or_else(|| "download".to_string());
1775    let extension = path
1776        .extension()
1777        .map(|extension| format!(".{}", extension.to_string_lossy()))
1778        .unwrap_or_default();
1779    for index in 1..10_000 {
1780        let candidate = path.with_file_name(format!("{stem}-{index}{extension}"));
1781        if !tokio::fs::try_exists(&candidate).await? {
1782            return Ok(candidate);
1783        }
1784    }
1785    Err(Error::Backend(
1786        "could not find a free destination name".to_string(),
1787    ))
1788}
1789
1790fn now_ms() -> u64 {
1791    SystemTime::now()
1792        .duration_since(UNIX_EPOCH)
1793        .map(|duration| duration.as_millis() as u64)
1794        .unwrap_or(0)
1795}
1796
1797async fn ensure_parent(path: &Path) -> Result<()> {
1798    if let Some(parent) = path.parent()
1799        && !parent.as_os_str().is_empty()
1800    {
1801        tokio::fs::create_dir_all(parent).await?;
1802    }
1803    Ok(())
1804}
1805
1806async fn remove_if_exists(path: &Path) -> Result<()> {
1807    match tokio::fs::remove_file(path).await {
1808        Ok(()) => Ok(()),
1809        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1810        Err(error) => Err(Error::Io(error)),
1811    }
1812}
1813
1814async fn checkpoint_single(
1815    request: &DownloadRequest,
1816    task_resume: &mut TaskResume,
1817    key: &str,
1818    file_resume: &mut FileResume,
1819    offset: u64,
1820) -> Result<()> {
1821    file_resume.completed = if offset > 0 {
1822        vec![shaul_core::ByteRange::new(0, offset)?]
1823    } else {
1824        Vec::new()
1825    };
1826    file_resume.normalize();
1827    task_resume
1828        .files
1829        .insert(key.to_string(), file_resume.clone());
1830    request.checkpoint.save(task_resume).await
1831}
1832
1833fn spawn_progress_reporter(
1834    events: EventSink,
1835    task_id: TaskId,
1836    file: String,
1837    total: Option<u64>,
1838    downloaded: Arc<AtomicU64>,
1839    stop: CancellationToken,
1840) -> tokio::task::JoinHandle<()> {
1841    tokio::spawn(async move {
1842        let mut ticker = tokio::time::interval(Duration::from_millis(100));
1843        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1844        loop {
1845            tokio::select! {
1846                _ = ticker.tick() => {
1847                    events.emit(EventKind::Progress {
1848                        task_id,
1849                        file: file.clone(),
1850                        downloaded_bytes: downloaded.load(Ordering::Relaxed),
1851                        total_bytes: total,
1852                    });
1853                }
1854                _ = stop.cancelled() => {
1855                    events.emit(EventKind::Progress {
1856                        task_id,
1857                        file,
1858                        downloaded_bytes: downloaded.load(Ordering::Relaxed),
1859                        total_bytes: total,
1860                    });
1861                    break;
1862                }
1863            }
1864        }
1865    })
1866}
1867
1868#[cfg(test)]
1869mod tests {
1870    use super::*;
1871
1872    fn validator(size: Option<u64>, etag: Option<&str>) -> Validator {
1873        Validator {
1874            etag: etag.map(str::to_string),
1875            last_modified: None,
1876            size,
1877        }
1878    }
1879
1880    #[test]
1881    fn mirror_compatibility_requires_matching_size() {
1882        assert!(mirror_compatible(
1883            &validator(Some(10), Some("\"a\"")),
1884            &validator(Some(10), Some("\"a\"")),
1885            false
1886        ));
1887        assert!(!mirror_compatible(
1888            &validator(Some(10), None),
1889            &validator(Some(11), None),
1890            true
1891        ));
1892    }
1893
1894    #[test]
1895    fn mirror_compatibility_uses_checksum_when_size_is_unknown() {
1896        assert!(!mirror_compatible(
1897            &validator(None, None),
1898            &validator(None, None),
1899            false
1900        ));
1901        assert!(mirror_compatible(
1902            &validator(None, None),
1903            &validator(None, None),
1904            true
1905        ));
1906    }
1907
1908    #[test]
1909    fn backoff_is_jittered_but_bounded() {
1910        let delay = backoff(0);
1911        assert!(delay >= Duration::from_millis(250));
1912        assert!(delay <= Duration::from_millis(312));
1913    }
1914
1915    #[test]
1916    fn mirror_compatibility_rejects_different_strong_etags() {
1917        assert!(!mirror_compatible(
1918            &validator(Some(10), Some("\"a\"")),
1919            &validator(Some(10), Some("\"b\"")),
1920            false
1921        ));
1922        assert!(mirror_compatible(
1923            &validator(Some(10), Some("\"a\"")),
1924            &validator(Some(10), Some("\"b\"")),
1925            true
1926        ));
1927    }
1928}