Skip to main content

shaul_http/
backend.rs

1use std::collections::{BTreeMap, VecDeque};
2use std::path::{Path, PathBuf};
3use std::sync::Arc;
4use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use async_trait::async_trait;
8use base64::Engine;
9use futures_util::StreamExt;
10use reqwest::header::{
11    ACCEPT_ENCODING, AUTHORIZATION, COOKIE, HeaderMap, HeaderName, HeaderValue, LOCATION,
12};
13use reqwest::redirect::Policy;
14use reqwest::{Client, Response, StatusCode};
15use shaul_core::{
16    BackendCapabilities, Checksum, ChecksumAlgorithm, Config, DownloadBackend, DownloadOutcome,
17    DownloadRequest, Error, EventKind, EventSink, FileOutcome, FileResume, Header, Resource,
18    ResourceFile, ResourceKind, Result, TaskId, TaskResume, Validator, VerifyMode, completed_bytes,
19    merge_ranges, missing_ranges, part_path, sanitize_file_name, split_range,
20};
21use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
22use tokio::sync::Mutex;
23use tokio_util::sync::CancellationToken;
24use url::Url;
25
26use crate::filename::parse_content_disposition;
27
28const MAX_REDIRECTS: usize = 10;
29
30const CAPTURE_HEADERS: &[&str] = &[
31    "content-disposition",
32    "etag",
33    "last-modified",
34    "content-length",
35    "accept-ranges",
36    "content-range",
37    "digest",
38    "content-md5",
39    "x-linked-etag",
40    "x-linked-size",
41    "x-repo-commit",
42    "x-xet-hash",
43    "x-xet-token-expiration",
44    "x-xet-refresh-route",
45    "x-error-code",
46];
47
48const RESERVED_HEADERS: &[&str] = &[
49    "range",
50    "if-range",
51    "host",
52    "content-length",
53    "accept-encoding",
54    "connection",
55    "transfer-encoding",
56];
57
58#[derive(Clone)]
59pub struct HttpBackend {
60    client: Client,
61}
62
63impl HttpBackend {
64    pub fn new(config: &Config) -> Result<Self> {
65        let client = Client::builder()
66            .redirect(Policy::none())
67            .connect_timeout(Duration::from_secs(
68                config.network.connect_timeout_secs.max(1),
69            ))
70            .pool_max_idle_per_host(config.download.per_host.max(1))
71            .user_agent(config.network.user_agent.clone())
72            .build()
73            .map_err(|error| Error::Backend(format!("failed to build HTTP client: {error}")))?;
74        Ok(Self { client })
75    }
76
77    /// Download one resource file, sharing the caller's task resume. This is
78    /// public so protocol-specific backends (Hub) can reuse the segmented,
79    /// resumable HTTP transfer without duplicating it.
80    pub async fn download_file(
81        &self,
82        request: &DownloadRequest,
83        file: &ResourceFile,
84        task_resume: &mut TaskResume,
85    ) -> Result<FileOutcome> {
86        let key = file.url.as_str().to_string();
87
88        if let Some(stored) = task_resume.files.get(&key).cloned()
89            && stored.is_complete()
90        {
91            let final_path = manifest_path(&request.destination, &stored.final_path);
92            let part = manifest_path(&request.destination, &stored.part_path);
93            if tokio::fs::try_exists(&final_path).await? {
94                let size = tokio::fs::metadata(&final_path).await?.len();
95                let relative = relative_path(&request.destination, &final_path);
96                request.events.emit(EventKind::FileStarted {
97                    task_id: request.task_id,
98                    file: relative.clone(),
99                    total_bytes: Some(size),
100                });
101                request.events.emit(EventKind::Progress {
102                    task_id: request.task_id,
103                    file: relative.clone(),
104                    downloaded_bytes: size,
105                    total_bytes: Some(size),
106                });
107                return Ok(FileOutcome::new(
108                    final_path,
109                    relative,
110                    size,
111                    stored.checksum.or(request.options.checksum.clone()),
112                    false,
113                ));
114            }
115            if tokio::fs::try_exists(&part).await? {
116                ensure_parent(&final_path).await?;
117                remove_if_exists(&final_path).await?;
118                tokio::fs::rename(&part, &final_path).await?;
119                let size = tokio::fs::metadata(&final_path).await?.len();
120                let relative = relative_path(&request.destination, &final_path);
121                request.events.emit(EventKind::FileStarted {
122                    task_id: request.task_id,
123                    file: relative.clone(),
124                    total_bytes: Some(size),
125                });
126                return Ok(FileOutcome::new(
127                    final_path,
128                    relative,
129                    size,
130                    stored.checksum.or(request.options.checksum.clone()),
131                    false,
132                ));
133            }
134        }
135
136        let probe = self.probe(request, file).await?;
137        let total = probe.total_size;
138        let validator = probe.validator();
139        let checksum = request
140            .options
141            .checksum
142            .clone()
143            .or_else(|| probe.checksum.clone())
144            .or_else(|| file.checksum.clone());
145        if request.options.verify == VerifyMode::Require && checksum.is_none() {
146            return Err(Error::ChecksumMissing);
147        }
148
149        let default_name = file
150            .path
151            .rsplit('/')
152            .next()
153            .map(sanitize_file_name)
154            .unwrap_or_else(|| "download".to_string());
155        let (mut final_path, relative) =
156            resolve_final_path(request, file, probe.filename.as_deref(), &default_name)?;
157
158        let stored = task_resume.files.get(&key).cloned();
159        let final_for_manifest = path_for_manifest(&request.destination, &final_path);
160        let stored_matches_path = stored
161            .as_ref()
162            .map(|stored| stored.final_path == final_for_manifest)
163            .unwrap_or(false);
164        let resume_valid = request.options.resume
165            && stored
166                .as_ref()
167                .map(|stored| {
168                    stored_matches_path
169                        && !stored.validator.is_empty()
170                        && stored.validator.matches(&validator)
171                        && match (stored.total_size, total) {
172                            (Some(old), Some(new)) => old == new,
173                            _ => true,
174                        }
175                })
176                .unwrap_or(false);
177
178        if stored.is_some() && !resume_valid {
179            request.events.emit(EventKind::Warning {
180                task_id: Some(request.task_id),
181                message: "resume state is no longer valid; restarting this file".to_string(),
182            });
183        }
184
185        final_path = apply_conflict_policy(final_path, &request.options).await?;
186        let part_name = part_path(&final_path);
187        let part_for_manifest = path_for_manifest(&request.destination, &part_name);
188
189        let mut file_resume = FileResume {
190            url: key.clone(),
191            final_path: path_for_manifest(&request.destination, &final_path),
192            part_path: part_for_manifest.clone(),
193            validator: validator.clone(),
194            total_size: total,
195            segment_size: request.options.segment_size,
196            completed: if resume_valid {
197                stored
198                    .as_ref()
199                    .map(|stored| stored.completed.clone())
200                    .unwrap_or_default()
201            } else {
202                Vec::new()
203            },
204            checksum: checksum.clone(),
205            metadata: {
206                let mut metadata = probe.headers.clone();
207                metadata.insert("final-url".to_string(), redacted_url(&probe.final_url));
208                metadata
209            },
210            updated_at_ms: now_ms(),
211        };
212
213        // Reconcile existing partial data with the checkpoint.
214        let existing_part_len = tokio::fs::metadata(&part_name)
215            .await
216            .ok()
217            .map(|metadata| metadata.len());
218        if !resume_valid {
219            file_resume.completed.clear();
220        } else if let Some(len) = existing_part_len {
221            let limit = total.map(|total| total.min(len)).unwrap_or(len);
222            file_resume.completed = shaul_core::range::clamp_to_len(&file_resume.completed, limit);
223        } else {
224            file_resume.completed.clear();
225        }
226        file_resume.normalize();
227
228        ensure_parent(&final_path).await?;
229        if let Some(total) = total {
230            let file = tokio::fs::OpenOptions::new()
231                .create(true)
232                .write(true)
233                .truncate(false)
234                .open(&part_name)
235                .await?;
236            if !resume_valid {
237                file.set_len(0).await?;
238            }
239            file.set_len(total).await?;
240            file.sync_all().await?;
241        } else if !resume_valid {
242            tokio::fs::OpenOptions::new()
243                .create(true)
244                .write(true)
245                .truncate(true)
246                .open(&part_name)
247                .await?;
248        } else if !tokio::fs::try_exists(&part_name).await? {
249            tokio::fs::File::create(&part_name).await?;
250            file_resume.completed.clear();
251        }
252
253        task_resume.files.insert(key.clone(), file_resume.clone());
254        request.checkpoint.save(task_resume).await?;
255
256        let initial = completed_bytes(&file_resume.completed);
257        let progress = Arc::new(AtomicU64::new(initial));
258        request.events.emit(EventKind::FileStarted {
259            task_id: request.task_id,
260            file: relative.clone(),
261            total_bytes: total,
262        });
263
264        let progress_stop = CancellationToken::new();
265        let reporter = spawn_progress_reporter(
266            request.events.clone(),
267            request.task_id,
268            relative.clone(),
269            total,
270            progress.clone(),
271            progress_stop.clone(),
272        );
273
274        let transfer_result = if let Some(0) = total {
275            // Empty file: the pre-allocated part is already the final content.
276            task_resume.files.insert(key.clone(), file_resume.clone());
277            request.checkpoint.save(task_resume).await?;
278            Ok(file_resume.clone())
279        } else if probe.range_supported
280            && total.map(|total| total > 0).unwrap_or(false)
281            && request.options.connections > 1
282        {
283            self.transfer_segmented(
284                request,
285                file,
286                &part_name,
287                &key,
288                file_resume.clone(),
289                task_resume.clone(),
290                progress.clone(),
291                total,
292            )
293            .await
294        } else {
295            self.transfer_single(
296                request,
297                file,
298                &part_name,
299                &key,
300                file_resume.clone(),
301                task_resume.clone(),
302                progress.clone(),
303                probe.range_supported,
304            )
305            .await
306        };
307
308        progress_stop.cancel();
309        let _ = reporter.await;
310
311        let mut final_resume = match transfer_result {
312            Ok(resume) => resume,
313            Err(error) => return Err(error),
314        };
315
316        if let Some(total) = final_resume.total_size
317            && total > 0
318        {
319            final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
320        }
321        final_resume.normalize();
322        request.events.emit(EventKind::Progress {
323            task_id: request.task_id,
324            file: relative.clone(),
325            downloaded_bytes: final_resume.completed_bytes(),
326            total_bytes: final_resume.total_size,
327        });
328        task_resume.files.insert(key, final_resume.clone());
329        request.checkpoint.save(task_resume).await?;
330
331        ensure_parent(&final_path).await?;
332        remove_if_exists(&final_path).await?;
333        if tokio::fs::try_exists(&part_name).await? {
334            tokio::fs::rename(&part_name, &final_path).await?;
335        } else {
336            tokio::fs::write(&final_path, []).await?;
337        }
338        let size = tokio::fs::metadata(&final_path).await?.len();
339        Ok(FileOutcome::new(
340            final_path, relative, size, checksum, false,
341        ))
342    }
343
344    async fn probe(&self, request: &DownloadRequest, file: &ResourceFile) -> Result<Probe> {
345        let mut attempt = 0u32;
346        loop {
347            let headers = build_headers(&request.options, &file.headers, Some("bytes=0-0"), None)?;
348            let sent = self
349                .send_follow(&file.url, &headers, &request.options, &request.cancel)
350                .await;
351            match sent {
352                Err(Error::Cancelled) => return Err(Error::Cancelled),
353                Err(error) => {
354                    if attempt >= request.options.retries {
355                        return Err(error);
356                    }
357                    sleep_or_cancel(backoff(attempt), &request.cancel).await?;
358                    attempt += 1;
359                }
360                Ok((response, captured, final_url)) => {
361                    let status = response.status();
362                    let retry_after = retry_after(response.headers());
363                    if status.is_server_error()
364                        || status == StatusCode::REQUEST_TIMEOUT
365                        || status == StatusCode::TOO_MANY_REQUESTS
366                    {
367                        drop(response);
368                        if attempt >= request.options.retries {
369                            return Err(http_error(status, &captured));
370                        }
371                        sleep_or_cancel(
372                            retry_after.unwrap_or_else(|| backoff(attempt)),
373                            &request.cancel,
374                        )
375                        .await?;
376                        attempt += 1;
377                        continue;
378                    }
379                    if matches!(
380                        status,
381                        StatusCode::UNAUTHORIZED
382                            | StatusCode::FORBIDDEN
383                            | StatusCode::NOT_FOUND
384                            | StatusCode::GONE
385                    ) {
386                        return Err(http_error(status, &captured));
387                    }
388
389                    let content_range = captured
390                        .get("content-range")
391                        .and_then(|value| parse_content_range(value));
392                    let linked_size = captured
393                        .get("x-linked-size")
394                        .and_then(|value| parse_u64(value));
395                    let content_length = captured
396                        .get("content-length")
397                        .and_then(|value| parse_u64(value));
398                    let (range_supported, total_size) = match status {
399                        StatusCode::PARTIAL_CONTENT => (
400                            true,
401                            content_range
402                                .and_then(|(_, _, total)| total)
403                                .or(linked_size),
404                        ),
405                        StatusCode::OK => match content_range {
406                            // Some servers answer a ranged request with 200 +
407                            // Content-Range (ModelScope does this). The body is
408                            // still a partial range, so use the total from
409                            // Content-Range rather than the range length in
410                            // Content-Length.
411                            Some((_, _, total)) => (true, total.or(linked_size)),
412                            None => (false, linked_size.or(content_length)),
413                        },
414                        StatusCode::RANGE_NOT_SATISFIABLE => {
415                            (true, content_range.and_then(|(_, _, total)| total))
416                        }
417                        _ => {
418                            return Err(Error::Backend(format!(
419                                "unexpected HTTP status during probe: {status}"
420                            )));
421                        }
422                    };
423
424                    let checksum = captured
425                        .get("x-linked-etag")
426                        .and_then(|value| Checksum::from_etag(value))
427                        .or_else(|| parse_digest(&captured));
428                    let etag = captured
429                        .get("etag")
430                        .cloned()
431                        .or_else(|| captured.get("x-linked-etag").cloned());
432                    let last_modified = captured.get("last-modified").cloned();
433                    let filename = captured
434                        .get("content-disposition")
435                        .and_then(|value| parse_content_disposition(value));
436                    let mut headers = captured;
437                    if let Some(total) = total_size {
438                        headers.insert("total-size".to_string(), total.to_string());
439                    }
440                    return Ok(Probe {
441                        final_url,
442                        total_size,
443                        range_supported,
444                        filename,
445                        etag,
446                        last_modified,
447                        checksum,
448                        headers,
449                    });
450                }
451            }
452        }
453    }
454
455    async fn send_follow(
456        &self,
457        url: &Url,
458        headers: &HeaderMap,
459        options: &shaul_core::ResolvedOptions,
460        cancel: &CancellationToken,
461    ) -> Result<(Response, BTreeMap<String, String>, Url)> {
462        let mut current = url.clone();
463        let mut headers = headers.clone();
464        let mut captured = BTreeMap::new();
465
466        for _ in 0..MAX_REDIRECTS {
467            let response = tokio::select! {
468                _ = cancel.cancelled() => return Err(Error::Cancelled),
469                response = self.client.get(current.clone()).headers(headers.clone()).send() => {
470                    response.map_err(|error| Error::Backend(format!("request failed: {error}")))?
471                }
472            };
473            capture_headers(&response, &mut captured);
474            let status = response.status();
475            if matches!(
476                status,
477                StatusCode::MOVED_PERMANENTLY
478                    | StatusCode::FOUND
479                    | StatusCode::SEE_OTHER
480                    | StatusCode::TEMPORARY_REDIRECT
481                    | StatusCode::PERMANENT_REDIRECT
482            ) {
483                let location = response
484                    .headers()
485                    .get(LOCATION)
486                    .and_then(|value| value.to_str().ok())
487                    .ok_or_else(|| {
488                        Error::Backend("redirect response missing Location".to_string())
489                    })?;
490                let next = current
491                    .join(location)
492                    .map_err(|error| Error::Backend(format!("invalid redirect URL: {error}")))?;
493                if !same_origin(&current, &next) && !options.request.allow_cross_origin_auth {
494                    for name in options.request.sensitive_header_names() {
495                        headers.remove(name.as_str());
496                    }
497                    headers.remove(AUTHORIZATION);
498                    headers.remove(COOKIE);
499                }
500                current = next;
501                continue;
502            }
503            return Ok((response, captured, current));
504        }
505        Err(Error::Backend("too many redirects".to_string()))
506    }
507
508    #[allow(clippy::too_many_arguments)]
509    async fn transfer_segmented(
510        &self,
511        request: &DownloadRequest,
512        file: &ResourceFile,
513        part_name: &Path,
514        key: &str,
515        file_resume: FileResume,
516        task_resume: TaskResume,
517        progress: Arc<AtomicU64>,
518        total: Option<u64>,
519    ) -> Result<FileResume> {
520        let total =
521            total.ok_or_else(|| Error::Backend("segmented transfer without size".to_string()))?;
522        let missing = missing_ranges(total, &file_resume.completed);
523        let mut chunks = Vec::new();
524        for range in missing {
525            chunks.extend(split_range(range, request.options.segment_size));
526        }
527        if chunks.is_empty() {
528            return Ok(file_resume);
529        }
530
531        let mut state = task_resume.clone();
532        state.files.insert(key.to_string(), file_resume.clone());
533        let resume_state = Arc::new(Mutex::new(state));
534        let checkpoint_lock = Arc::new(Mutex::new(()));
535        let queue = Arc::new(std::sync::Mutex::new(VecDeque::from(chunks)));
536        let range_broken = Arc::new(AtomicBool::new(false));
537        let child_cancel = request.cancel.child_token();
538        let if_range = strong_if_range(&file_resume.validator);
539        let workers = request
540            .options
541            .connections
542            .min(queue.lock().map(|queue| queue.len()).unwrap_or(1))
543            .max(1);
544
545        let mut set = tokio::task::JoinSet::new();
546        for _ in 0..workers {
547            let backend = self.clone();
548            let request = request.clone();
549            let file = file.clone();
550            let part_name = part_name.to_path_buf();
551            let key = key.to_string();
552            let queue = queue.clone();
553            let resume_state = resume_state.clone();
554            let checkpoint_lock = checkpoint_lock.clone();
555            let range_broken = range_broken.clone();
556            let progress = progress.clone();
557            let cancel = child_cancel.clone();
558            let if_range = if_range.clone();
559            set.spawn(async move {
560                backend
561                    .segment_worker(
562                        &request,
563                        &file,
564                        &part_name,
565                        &key,
566                        queue,
567                        resume_state,
568                        checkpoint_lock,
569                        range_broken,
570                        progress,
571                        cancel,
572                        total,
573                        if_range,
574                    )
575                    .await
576            });
577        }
578
579        let mut first_error = None;
580        while let Some(joined) = set.join_next().await {
581            match joined {
582                Ok(Ok(())) => {}
583                Ok(Err(FetchError::Cancelled)) => {}
584                Ok(Err(FetchError::RangeBroken)) => {}
585                Ok(Err(FetchError::Fatal(error))) => {
586                    if first_error.is_none() {
587                        first_error = Some(error);
588                    }
589                    child_cancel.cancel();
590                }
591                Ok(Err(FetchError::Retryable(message))) => {
592                    if first_error.is_none() {
593                        first_error = Some(Error::Backend(message));
594                    }
595                    child_cancel.cancel();
596                }
597                Err(error) => {
598                    if first_error.is_none() {
599                        first_error = Some(Error::Other(format!("segment worker failed: {error}")));
600                    }
601                    child_cancel.cancel();
602                }
603            }
604        }
605
606        if let Some(error) = first_error {
607            return Err(error);
608        }
609
610        if range_broken.load(Ordering::SeqCst) {
611            request.events.emit(EventKind::Warning {
612                task_id: Some(request.task_id),
613                message: "server ignored a ranged request; falling back to a single stream"
614                    .to_string(),
615            });
616            let file_handle = tokio::fs::OpenOptions::new()
617                .create(true)
618                .write(true)
619                .truncate(false)
620                .open(part_name)
621                .await?;
622            file_handle.set_len(0).await?;
623            file_handle.sync_all().await?;
624            progress.store(0, Ordering::Relaxed);
625            let mut reset = file_resume;
626            reset.completed.clear();
627            reset.normalize();
628            return self
629                .transfer_single(
630                    request,
631                    file,
632                    part_name,
633                    key,
634                    reset,
635                    task_resume,
636                    progress,
637                    false,
638                )
639                .await;
640        }
641
642        let final_state = resume_state.lock().await.clone();
643        let mut final_resume = final_state
644            .files
645            .get(key)
646            .cloned()
647            .ok_or_else(|| Error::Backend("missing resume state after transfer".to_string()))?;
648        final_resume.completed = vec![shaul_core::ByteRange::new(0, total)?];
649        final_resume.normalize();
650        let mut snapshot = final_state;
651        snapshot.files.insert(key.to_string(), final_resume.clone());
652        request.checkpoint.save(&snapshot).await?;
653        Ok(final_resume)
654    }
655
656    #[allow(clippy::too_many_arguments)]
657    async fn segment_worker(
658        &self,
659        request: &DownloadRequest,
660        file: &ResourceFile,
661        part_name: &Path,
662        key: &str,
663        queue: Arc<std::sync::Mutex<VecDeque<shaul_core::ByteRange>>>,
664        resume_state: Arc<Mutex<TaskResume>>,
665        checkpoint_lock: Arc<Mutex<()>>,
666        range_broken: Arc<AtomicBool>,
667        progress: Arc<AtomicU64>,
668        cancel: CancellationToken,
669        total: u64,
670        if_range: Option<String>,
671    ) -> std::result::Result<(), FetchError> {
672        let mut part = tokio::fs::OpenOptions::new()
673            .write(true)
674            .open(part_name)
675            .await
676            .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
677        loop {
678            if cancel.is_cancelled() || range_broken.load(Ordering::SeqCst) {
679                break;
680            }
681            let range = {
682                let mut queue = queue
683                    .lock()
684                    .map_err(|_| FetchError::Fatal(Error::Other("work queue poisoned".into())))?;
685                queue.pop_front()
686            };
687            let Some(range) = range else {
688                break;
689            };
690            match self
691                .fetch_range(
692                    request,
693                    file,
694                    range,
695                    &mut part,
696                    &progress,
697                    &cancel,
698                    total,
699                    if_range.as_deref(),
700                )
701                .await
702            {
703                Ok(()) => {
704                    let _guard = checkpoint_lock.lock().await;
705                    let snapshot = {
706                        let mut state = resume_state.lock().await;
707                        if let Some(entry) = state.files.get_mut(key) {
708                            entry.completed.push(range);
709                            entry.normalize();
710                        }
711                        state.clone()
712                    };
713                    request
714                        .checkpoint
715                        .save(&snapshot)
716                        .await
717                        .map_err(FetchError::Fatal)?;
718                }
719                Err(FetchError::Cancelled) => break,
720                Err(FetchError::RangeBroken) => {
721                    range_broken.store(true, Ordering::SeqCst);
722                    cancel.cancel();
723                    break;
724                }
725                Err(error) => {
726                    cancel.cancel();
727                    return Err(error);
728                }
729            }
730        }
731        part.flush()
732            .await
733            .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
734        Ok(())
735    }
736
737    #[allow(clippy::too_many_arguments)]
738    async fn fetch_range(
739        &self,
740        request: &DownloadRequest,
741        file: &ResourceFile,
742        range: shaul_core::ByteRange,
743        part: &mut tokio::fs::File,
744        progress: &AtomicU64,
745        cancel: &CancellationToken,
746        total: u64,
747        if_range: Option<&str>,
748    ) -> std::result::Result<(), FetchError> {
749        let mut attempt = 0u32;
750        loop {
751            if cancel.is_cancelled() {
752                return Err(FetchError::Cancelled);
753            }
754            let range_header = format!("bytes={}-{}", range.start, range.end - 1);
755            let headers = build_headers(
756                &request.options,
757                &file.headers,
758                Some(&range_header),
759                if_range,
760            )
761            .map_err(FetchError::Fatal)?;
762            let sent = self
763                .send_follow(&file.url, &headers, &request.options, cancel)
764                .await;
765            let (response, captured, _) = match sent {
766                Err(Error::Cancelled) => return Err(FetchError::Cancelled),
767                Err(error) => {
768                    if attempt >= request.options.retries {
769                        return Err(FetchError::Fatal(error));
770                    }
771                    sleep_or_cancel(backoff(attempt), cancel)
772                        .await
773                        .map_err(FetchError::Fatal)?;
774                    attempt += 1;
775                    continue;
776                }
777                Ok(response) => response,
778            };
779            let status = response.status();
780            let retry_after = retry_after(response.headers());
781
782            if status == StatusCode::RANGE_NOT_SATISFIABLE {
783                return Err(FetchError::Fatal(Error::Backend(format!(
784                    "range {} is not satisfiable",
785                    range_header
786                ))));
787            }
788            if status.is_server_error()
789                || status == StatusCode::REQUEST_TIMEOUT
790                || status == StatusCode::TOO_MANY_REQUESTS
791            {
792                drop(response);
793                if attempt >= request.options.retries {
794                    return Err(FetchError::Fatal(http_error(status, &captured)));
795                }
796                sleep_or_cancel(retry_after.unwrap_or_else(|| backoff(attempt)), cancel)
797                    .await
798                    .map_err(FetchError::Fatal)?;
799                attempt += 1;
800                continue;
801            }
802
803            let content_range = captured
804                .get("content-range")
805                .and_then(|value| parse_content_range(value));
806            if status == StatusCode::OK && content_range.is_none() {
807                return Err(FetchError::RangeBroken);
808            }
809            let is_partial = status == StatusCode::PARTIAL_CONTENT
810                || (status == StatusCode::OK && content_range.is_some());
811            if !is_partial {
812                return Err(FetchError::Fatal(http_error(status, &captured)));
813            }
814            let parsed = content_range.ok_or_else(|| {
815                FetchError::Fatal(Error::Backend(
816                    "ranged response missing a valid Content-Range".to_string(),
817                ))
818            })?;
819            if parsed.0 != range.start || parsed.1 + 1 != range.end {
820                return Err(FetchError::Fatal(Error::Backend(format!(
821                    "server returned range {}-{}, expected {}-{}",
822                    parsed.0,
823                    parsed.1,
824                    range.start,
825                    range.end - 1
826                ))));
827            }
828            if let Some(server_total) = parsed.2
829                && server_total != total
830            {
831                return Err(FetchError::Fatal(Error::Backend(format!(
832                    "file size changed from {total} to {server_total}"
833                ))));
834            }
835
836            part.seek(SeekFrom::Start(range.start))
837                .await
838                .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
839            let mut written = 0u64;
840            let mut stream = response.bytes_stream();
841            let retry_message;
842            loop {
843                let next = tokio::select! {
844                    _ = cancel.cancelled() => return Err(FetchError::Cancelled),
845                    next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
846                };
847                match next {
848                    Ok(Some(Ok(chunk))) => {
849                        if written + chunk.len() as u64 > range.len() {
850                            retry_message =
851                                Some("server sent more data than requested".to_string());
852                            break;
853                        }
854                        part.write_all(&chunk)
855                            .await
856                            .map_err(|error| FetchError::Fatal(Error::Io(error)))?;
857                        written += chunk.len() as u64;
858                        progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
859                    }
860                    Ok(Some(Err(error))) => {
861                        retry_message = Some(format!("stream error: {error}"));
862                        break;
863                    }
864                    Ok(None) => {
865                        if written == range.len() {
866                            return Ok(());
867                        }
868                        retry_message = Some(format!(
869                            "short ranged response: {written} of {} bytes",
870                            range.len()
871                        ));
872                        break;
873                    }
874                    Err(_) => {
875                        retry_message = Some("read timed out".to_string());
876                        break;
877                    }
878                }
879            }
880
881            // A retry rewrites this range from the beginning; remove the
882            // partial bytes from the progress counter first.
883            let current = progress.load(Ordering::Relaxed);
884            progress.fetch_sub(written.min(current), Ordering::Relaxed);
885            if attempt >= request.options.retries {
886                return Err(FetchError::Retryable(
887                    retry_message.unwrap_or_else(|| "range transfer failed".to_string()),
888                ));
889            }
890            sleep_or_cancel(backoff(attempt), cancel)
891                .await
892                .map_err(FetchError::Fatal)?;
893            attempt += 1;
894        }
895    }
896
897    #[allow(clippy::too_many_arguments)]
898    async fn transfer_single(
899        &self,
900        request: &DownloadRequest,
901        file: &ResourceFile,
902        part_name: &Path,
903        key: &str,
904        mut file_resume: FileResume,
905        mut task_resume: TaskResume,
906        progress: Arc<AtomicU64>,
907        range_supported: bool,
908    ) -> Result<FileResume> {
909        let mut offset = if request.options.resume && range_supported {
910            contiguous_end(&file_resume.completed)
911        } else {
912            0
913        };
914        if file_resume
915            .total_size
916            .map(|total| offset >= total)
917            .unwrap_or(false)
918        {
919            return Ok(file_resume);
920        }
921
922        let mut part = tokio::fs::OpenOptions::new()
923            .create(true)
924            .write(true)
925            .truncate(false)
926            .open(part_name)
927            .await?;
928        if offset == 0 {
929            part.set_len(0).await?;
930        }
931        part.seek(SeekFrom::Start(offset)).await?;
932
933        let mut attempt = 0u32;
934        loop {
935            if request.cancel.is_cancelled() {
936                checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
937                return Err(Error::Cancelled);
938            }
939            let range = if offset > 0 && range_supported {
940                Some(format!("bytes={offset}-"))
941            } else {
942                None
943            };
944            let if_range = if offset > 0 {
945                strong_if_range(&file_resume.validator)
946            } else {
947                None
948            };
949            let headers = build_headers(
950                &request.options,
951                &file.headers,
952                range.as_deref(),
953                if_range.as_deref(),
954            )?;
955            let sent = self
956                .send_follow(&file.url, &headers, &request.options, &request.cancel)
957                .await;
958            let (response, captured, _) = match sent {
959                Err(Error::Cancelled) => {
960                    checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
961                        .await?;
962                    return Err(Error::Cancelled);
963                }
964                Err(error) => {
965                    checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset)
966                        .await?;
967                    if attempt >= request.options.retries {
968                        return Err(error);
969                    }
970                    sleep_or_cancel(backoff(attempt), &request.cancel).await?;
971                    attempt += 1;
972                    continue;
973                }
974                Ok(response) => response,
975            };
976            let status = response.status();
977            let retry_after = retry_after(response.headers());
978
979            let content_range = captured
980                .get("content-range")
981                .and_then(|value| parse_content_range(value));
982            if status.is_server_error()
983                || status == StatusCode::REQUEST_TIMEOUT
984                || status == StatusCode::TOO_MANY_REQUESTS
985            {
986                drop(response);
987                checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
988                if attempt >= request.options.retries {
989                    return Err(http_error(status, &captured));
990                }
991                sleep_or_cancel(
992                    retry_after.unwrap_or_else(|| backoff(attempt)),
993                    &request.cancel,
994                )
995                .await?;
996                attempt += 1;
997                continue;
998            }
999
1000            let is_partial = status == StatusCode::PARTIAL_CONTENT
1001                || (status == StatusCode::OK && content_range.is_some());
1002            if is_partial {
1003                if let Some((start, _, _)) = content_range
1004                    && offset > 0
1005                    && start != offset
1006                {
1007                    return Err(Error::Backend(format!(
1008                        "server resumed at {start}, expected {offset}"
1009                    )));
1010                }
1011            } else if status == StatusCode::OK {
1012                if offset > 0 {
1013                    part.set_len(0).await?;
1014                    part.seek(SeekFrom::Start(0)).await?;
1015                    offset = 0;
1016                    progress.store(0, Ordering::Relaxed);
1017                }
1018            } else {
1019                return Err(http_error(status, &captured));
1020            }
1021
1022            part.seek(SeekFrom::Start(offset)).await?;
1023            let mut stream = response.bytes_stream();
1024            let mut since_checkpoint = 0u64;
1025            let retry_message;
1026            loop {
1027                let next = tokio::select! {
1028                    _ = request.cancel.cancelled() => {
1029                        checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1030                        part.sync_all().await?;
1031                        return Err(Error::Cancelled);
1032                    }
1033                    next = tokio::time::timeout(request.options.timeout, stream.next()) => next,
1034                };
1035                match next {
1036                    Ok(Some(Ok(chunk))) => {
1037                        if let Some(total) = file_resume.total_size
1038                            && offset + chunk.len() as u64 > total
1039                        {
1040                            return Err(Error::Backend(
1041                                "server sent more data than the known file size".to_string(),
1042                            ));
1043                        }
1044                        part.write_all(&chunk).await?;
1045                        offset += chunk.len() as u64;
1046                        progress.fetch_add(chunk.len() as u64, Ordering::Relaxed);
1047                        since_checkpoint += chunk.len() as u64;
1048                        if since_checkpoint >= request.options.segment_size {
1049                            checkpoint_single(
1050                                request,
1051                                &mut task_resume,
1052                                key,
1053                                &mut file_resume,
1054                                offset,
1055                            )
1056                            .await?;
1057                            since_checkpoint = 0;
1058                        }
1059                    }
1060                    Ok(Some(Err(error))) => {
1061                        retry_message = Some(format!("stream error: {error}"));
1062                        break;
1063                    }
1064                    Ok(None) => {
1065                        if let Some(total) = file_resume.total_size
1066                            && offset < total
1067                        {
1068                            retry_message =
1069                                Some(format!("short response: {offset} of {total} bytes"));
1070                            break;
1071                        }
1072                        part.flush().await?;
1073                        part.sync_all().await?;
1074                        file_resume.total_size = file_resume.total_size.or(Some(offset));
1075                        file_resume.completed = if offset > 0 {
1076                            vec![shaul_core::ByteRange::new(0, offset)?]
1077                        } else {
1078                            Vec::new()
1079                        };
1080                        file_resume.normalize();
1081                        task_resume
1082                            .files
1083                            .insert(key.to_string(), file_resume.clone());
1084                        request.checkpoint.save(&task_resume).await?;
1085                        return Ok(file_resume);
1086                    }
1087                    Err(_) => {
1088                        retry_message = Some("read timed out".to_string());
1089                        break;
1090                    }
1091                }
1092            }
1093
1094            part.flush().await?;
1095            checkpoint_single(request, &mut task_resume, key, &mut file_resume, offset).await?;
1096            if attempt >= request.options.retries {
1097                return Err(Error::Backend(
1098                    retry_message.unwrap_or_else(|| "download failed".to_string()),
1099                ));
1100            }
1101            sleep_or_cancel(backoff(attempt), &request.cancel).await?;
1102            attempt += 1;
1103        }
1104    }
1105}
1106
1107#[async_trait]
1108impl DownloadBackend for HttpBackend {
1109    fn name(&self) -> &'static str {
1110        "http"
1111    }
1112
1113    fn capabilities(&self) -> BackendCapabilities {
1114        BackendCapabilities::new(true, true, true)
1115    }
1116
1117    fn supports(&self, resource: &Resource) -> bool {
1118        matches!(&resource.kind, ResourceKind::Http)
1119    }
1120
1121    async fn download(&self, request: DownloadRequest) -> Result<DownloadOutcome> {
1122        let mut task_resume = request.resume.clone().unwrap_or_default();
1123        let mut outcomes = Vec::new();
1124        for file in &request.resource.files {
1125            if request.cancel.is_cancelled() {
1126                return Err(Error::Cancelled);
1127            }
1128            let outcome = self.download_file(&request, file, &mut task_resume).await?;
1129            outcomes.push(outcome);
1130            request.checkpoint.save(&task_resume).await?;
1131        }
1132        Ok(DownloadOutcome::Completed(outcomes))
1133    }
1134}
1135
1136struct Probe {
1137    final_url: Url,
1138    total_size: Option<u64>,
1139    range_supported: bool,
1140    filename: Option<String>,
1141    etag: Option<String>,
1142    last_modified: Option<String>,
1143    checksum: Option<Checksum>,
1144    headers: BTreeMap<String, String>,
1145}
1146
1147impl Probe {
1148    fn validator(&self) -> Validator {
1149        Validator {
1150            etag: self.etag.clone(),
1151            last_modified: self.last_modified.clone(),
1152            size: self.total_size,
1153        }
1154    }
1155}
1156
1157#[derive(Debug)]
1158enum FetchError {
1159    RangeBroken,
1160    Cancelled,
1161    Retryable(String),
1162    Fatal(Error),
1163}
1164
1165fn build_headers(
1166    options: &shaul_core::ResolvedOptions,
1167    file_headers: &[Header],
1168    range: Option<&str>,
1169    if_range: Option<&str>,
1170) -> Result<HeaderMap> {
1171    let mut headers = HeaderMap::new();
1172    for header in &options.request.headers {
1173        if !is_reserved(&header.name) {
1174            insert_header(
1175                &mut headers,
1176                &header.name,
1177                &shaul_core::expand_env(&header.value)?,
1178            )?;
1179        }
1180    }
1181    for header in file_headers {
1182        if !is_reserved(&header.name) {
1183            insert_header(
1184                &mut headers,
1185                &header.name,
1186                &shaul_core::expand_env(&header.value)?,
1187            )?;
1188        }
1189    }
1190
1191    if !headers.contains_key(AUTHORIZATION)
1192        && let Some(auth) = &options.request.auth
1193    {
1194        match auth {
1195            shaul_core::Auth::Bearer { token } => {
1196                insert_header(
1197                    &mut headers,
1198                    "authorization",
1199                    &format!("Bearer {}", shaul_core::expand_env(token)?),
1200                )?;
1201            }
1202            shaul_core::Auth::Basic { username, password } => {
1203                let encoded = base64::engine::general_purpose::STANDARD.encode(format!(
1204                    "{}:{}",
1205                    shaul_core::expand_env(username)?,
1206                    shaul_core::expand_env(password)?
1207                ));
1208                insert_header(&mut headers, "authorization", &format!("Basic {encoded}"))?;
1209            }
1210            shaul_core::Auth::Header(header) => {
1211                insert_header(
1212                    &mut headers,
1213                    &header.name,
1214                    &shaul_core::expand_env(&header.value)?,
1215                )?;
1216            }
1217        }
1218    }
1219
1220    headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity"));
1221    insert_header(&mut headers, "user-agent", &options.user_agent)?;
1222    if let Some(range) = range {
1223        insert_header(&mut headers, "range", range)?;
1224    }
1225    if let Some(if_range) = if_range {
1226        insert_header(&mut headers, "if-range", if_range)?;
1227    }
1228    Ok(headers)
1229}
1230
1231fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<()> {
1232    let name = HeaderName::from_bytes(name.as_bytes())
1233        .map_err(|_| Error::Config(format!("invalid header name: {name}")))?;
1234    let value = HeaderValue::from_str(value)
1235        .map_err(|_| Error::Config(format!("invalid value for header: {name}")))?;
1236    headers.insert(name, value);
1237    Ok(())
1238}
1239
1240fn is_reserved(name: &str) -> bool {
1241    RESERVED_HEADERS
1242        .iter()
1243        .any(|reserved| name.eq_ignore_ascii_case(reserved))
1244}
1245
1246fn capture_headers(response: &Response, captured: &mut BTreeMap<String, String>) {
1247    for name in CAPTURE_HEADERS {
1248        if let Some(value) = response.headers().get(*name)
1249            && let Ok(value) = value.to_str()
1250        {
1251            captured.insert((*name).to_string(), value.to_string());
1252        }
1253    }
1254}
1255
1256fn redacted_url(url: &Url) -> String {
1257    let mut url = url.clone();
1258    url.set_query(None);
1259    url.set_fragment(None);
1260    url.to_string()
1261}
1262
1263fn same_origin(left: &Url, right: &Url) -> bool {
1264    left.scheme() == right.scheme()
1265        && left.host_str() == right.host_str()
1266        && left.port_or_known_default() == right.port_or_known_default()
1267}
1268
1269fn parse_content_range(value: &str) -> Option<(u64, u64, Option<u64>)> {
1270    let rest = value.trim().strip_prefix("bytes")?.trim();
1271    let (range, total) = rest.split_once('/')?;
1272    let total = if total.trim() == "*" {
1273        None
1274    } else {
1275        Some(total.trim().parse::<u64>().ok()?)
1276    };
1277    if range.trim() == "*" {
1278        return Some((0, 0, total));
1279    }
1280    let (start, end) = range.split_once('-')?;
1281    Some((
1282        start.trim().parse::<u64>().ok()?,
1283        end.trim().parse::<u64>().ok()?,
1284        total,
1285    ))
1286}
1287
1288fn parse_u64(value: &str) -> Option<u64> {
1289    value.trim().parse::<u64>().ok()
1290}
1291
1292fn retry_after(headers: &HeaderMap) -> Option<Duration> {
1293    headers
1294        .get("retry-after")
1295        .and_then(|value| value.to_str().ok())
1296        .and_then(|value| value.trim().parse::<u64>().ok())
1297        .map(|seconds| Duration::from_secs(seconds.min(60)))
1298}
1299
1300fn backoff(attempt: u32) -> Duration {
1301    Duration::from_millis(250u64.saturating_mul(1u64 << attempt.min(5)))
1302}
1303
1304async fn sleep_or_cancel(duration: Duration, cancel: &CancellationToken) -> Result<()> {
1305    tokio::select! {
1306        _ = cancel.cancelled() => Err(Error::Cancelled),
1307        _ = tokio::time::sleep(duration) => Ok(()),
1308    }
1309}
1310
1311fn http_error(status: StatusCode, captured: &BTreeMap<String, String>) -> Error {
1312    let code = captured
1313        .get("x-error-code")
1314        .map(|code| format!(" ({code})"))
1315        .unwrap_or_default();
1316    Error::Backend(format!("HTTP {status}{code}"))
1317}
1318
1319fn parse_digest(captured: &BTreeMap<String, String>) -> Option<Checksum> {
1320    if let Some(value) = captured.get("digest") {
1321        for part in value.split(',') {
1322            let Some((algorithm, encoded)) = part.trim().split_once('=') else {
1323                continue;
1324            };
1325            let algorithm = match algorithm.trim().to_ascii_lowercase().as_str() {
1326                "sha-256" | "sha256" => ChecksumAlgorithm::Sha256,
1327                "sha-512" | "sha512" => ChecksumAlgorithm::Sha512,
1328                "sha" | "sha-1" | "sha1" => ChecksumAlgorithm::Sha1,
1329                "md5" => ChecksumAlgorithm::Md5,
1330                _ => continue,
1331            };
1332            if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(encoded.trim())
1333                && let Ok(checksum) = Checksum::new(algorithm, hex::encode(bytes))
1334            {
1335                return Some(checksum);
1336            }
1337        }
1338    }
1339    if let Some(value) = captured.get("content-md5")
1340        && let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(value.trim())
1341        && let Ok(checksum) = Checksum::new(ChecksumAlgorithm::Md5, hex::encode(bytes))
1342    {
1343        return Some(checksum);
1344    }
1345    None
1346}
1347
1348fn strong_if_range(validator: &Validator) -> Option<String> {
1349    if let Some(etag) = &validator.etag
1350        && !etag.starts_with("W/")
1351    {
1352        return Some(etag.clone());
1353    }
1354    validator.last_modified.clone()
1355}
1356
1357fn contiguous_end(completed: &[shaul_core::ByteRange]) -> u64 {
1358    let merged = merge_ranges(completed.to_vec());
1359    if merged.len() == 1 && merged[0].start == 0 {
1360        merged[0].end
1361    } else {
1362        0
1363    }
1364}
1365
1366fn manifest_path(destination: &Path, value: &str) -> PathBuf {
1367    let path = Path::new(value);
1368    if path.is_absolute() {
1369        path.to_path_buf()
1370    } else {
1371        destination.join(path)
1372    }
1373}
1374
1375fn path_for_manifest(destination: &Path, path: &Path) -> String {
1376    match path.strip_prefix(destination) {
1377        Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1378        Err(_) => path.to_string_lossy().to_string(),
1379    }
1380}
1381
1382fn relative_path(destination: &Path, path: &Path) -> String {
1383    match path.strip_prefix(destination) {
1384        Ok(relative) => relative.to_string_lossy().replace('\\', "/"),
1385        Err(_) => path
1386            .file_name()
1387            .map(|name| name.to_string_lossy().to_string())
1388            .unwrap_or_else(|| "download".to_string()),
1389    }
1390}
1391
1392fn resolve_final_path(
1393    request: &DownloadRequest,
1394    file: &ResourceFile,
1395    probe_filename: Option<&str>,
1396    default_name: &str,
1397) -> Result<(PathBuf, String)> {
1398    let single = request.resource.files.len() == 1;
1399    let response_name = if file.use_response_filename {
1400        probe_filename.map(sanitize_file_name)
1401    } else {
1402        None
1403    };
1404    let name = response_name
1405        .clone()
1406        .unwrap_or_else(|| default_name.to_string());
1407
1408    if let Some(output) = &request.options.output {
1409        if single {
1410            if output.is_dir() {
1411                let relative = if file.use_response_filename {
1412                    name.clone()
1413                } else {
1414                    file.path.clone()
1415                };
1416                let path = output.join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1417                return Ok((path.clone(), relative_path(&request.destination, &path)));
1418            }
1419            return Ok((output.clone(), relative_path(&request.destination, output)));
1420        }
1421        let path = output.join(file.path.replace('/', std::path::MAIN_SEPARATOR_STR));
1422        return Ok((path.clone(), relative_path(&request.destination, &path)));
1423    }
1424
1425    let relative = if single && file.use_response_filename {
1426        name
1427    } else {
1428        file.path.clone()
1429    };
1430    let path = request
1431        .destination
1432        .join(relative.replace('/', std::path::MAIN_SEPARATOR_STR));
1433    Ok((path.clone(), relative_path(&request.destination, &path)))
1434}
1435
1436async fn apply_conflict_policy(
1437    path: PathBuf,
1438    options: &shaul_core::ResolvedOptions,
1439) -> Result<PathBuf> {
1440    if !tokio::fs::try_exists(&path).await? {
1441        return Ok(path);
1442    }
1443    if options.overwrite {
1444        remove_if_exists(&path).await?;
1445        return Ok(path);
1446    }
1447    if options.rename_on_conflict {
1448        return unique_path(&path).await;
1449    }
1450    Err(Error::Backend(format!(
1451        "destination already exists: {} (use --force or --rename)",
1452        path.display()
1453    )))
1454}
1455
1456async fn unique_path(path: &Path) -> Result<PathBuf> {
1457    let stem = path
1458        .file_stem()
1459        .map(|stem| stem.to_string_lossy().to_string())
1460        .unwrap_or_else(|| "download".to_string());
1461    let extension = path
1462        .extension()
1463        .map(|extension| format!(".{}", extension.to_string_lossy()))
1464        .unwrap_or_default();
1465    for index in 1..10_000 {
1466        let candidate = path.with_file_name(format!("{stem}-{index}{extension}"));
1467        if !tokio::fs::try_exists(&candidate).await? {
1468            return Ok(candidate);
1469        }
1470    }
1471    Err(Error::Backend(
1472        "could not find a free destination name".to_string(),
1473    ))
1474}
1475
1476fn now_ms() -> u64 {
1477    SystemTime::now()
1478        .duration_since(UNIX_EPOCH)
1479        .map(|duration| duration.as_millis() as u64)
1480        .unwrap_or(0)
1481}
1482
1483async fn ensure_parent(path: &Path) -> Result<()> {
1484    if let Some(parent) = path.parent()
1485        && !parent.as_os_str().is_empty()
1486    {
1487        tokio::fs::create_dir_all(parent).await?;
1488    }
1489    Ok(())
1490}
1491
1492async fn remove_if_exists(path: &Path) -> Result<()> {
1493    match tokio::fs::remove_file(path).await {
1494        Ok(()) => Ok(()),
1495        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
1496        Err(error) => Err(Error::Io(error)),
1497    }
1498}
1499
1500async fn checkpoint_single(
1501    request: &DownloadRequest,
1502    task_resume: &mut TaskResume,
1503    key: &str,
1504    file_resume: &mut FileResume,
1505    offset: u64,
1506) -> Result<()> {
1507    file_resume.completed = if offset > 0 {
1508        vec![shaul_core::ByteRange::new(0, offset)?]
1509    } else {
1510        Vec::new()
1511    };
1512    file_resume.normalize();
1513    task_resume
1514        .files
1515        .insert(key.to_string(), file_resume.clone());
1516    request.checkpoint.save(task_resume).await
1517}
1518
1519fn spawn_progress_reporter(
1520    events: EventSink,
1521    task_id: TaskId,
1522    file: String,
1523    total: Option<u64>,
1524    downloaded: Arc<AtomicU64>,
1525    stop: CancellationToken,
1526) -> tokio::task::JoinHandle<()> {
1527    tokio::spawn(async move {
1528        let mut ticker = tokio::time::interval(Duration::from_millis(100));
1529        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1530        loop {
1531            tokio::select! {
1532                _ = ticker.tick() => {
1533                    events.emit(EventKind::Progress {
1534                        task_id,
1535                        file: file.clone(),
1536                        downloaded_bytes: downloaded.load(Ordering::Relaxed),
1537                        total_bytes: total,
1538                    });
1539                }
1540                _ = stop.cancelled() => {
1541                    events.emit(EventKind::Progress {
1542                        task_id,
1543                        file,
1544                        downloaded_bytes: downloaded.load(Ordering::Relaxed),
1545                        total_bytes: total,
1546                    });
1547                    break;
1548                }
1549            }
1550        }
1551    })
1552}