Skip to main content

grit_lib/
resumable.rs

1//! Transfers that survive a dropped connection: downloads resumed with HTTP
2//! `Range` requests, and uploads over the [TUS] resumable-upload protocol.
3//!
4//! Clone uses [`download_resumable`] for the bundle a server advertises with
5//! `bundle-uri` ([`crate::bundle_uri`]); push uses [`TusUpload`] to send a large
6//! bundle to the endpoint a server advertises with `pushBundle`
7//! ([`crate::push`]). Both keep their progress on disk, so a transfer that is
8//! interrupted (even by the process exiting) picks up where it stopped the next
9//! time it runs, and both retry a dropped connection a few times before giving
10//! up.
11//!
12//! Requests go through [`HttpClient::send`]; clients that do not implement it
13//! cannot make these transfers. Progress is reported both as the
14//! [`ProgressPhase::TransferringBundle`] phase and, in more detail, as
15//! [`ResumableEvent`]s.
16//!
17//! [TUS]: https://tus.io/protocols/resumable-upload
18
19use std::fs::{File, OpenOptions};
20use std::io::{Read, Seek, SeekFrom, Write};
21use std::path::{Path, PathBuf};
22
23use serde::{Deserialize, Serialize};
24
25use crate::fetch::Progress;
26use crate::progress::{CancelToken, ProgressPhase, ProgressUnit};
27use crate::transport::http::{HttpClient, HttpMethod, HttpRequest, HttpResponse};
28use crate::transport_path::redact_url_for_display;
29
30/// TUS protocol version this client and `grit serve` speak.
31pub const TUS_VERSION: &str = "1.0.0";
32
33/// Content type of a TUS `PATCH` body.
34pub const TUS_PATCH_CONTENT_TYPE: &str = "application/offset+octet-stream";
35
36/// Bytes sent per TUS `PATCH` request.
37const PATCH_CHUNK: usize = 4 * 1024 * 1024;
38
39/// Bytes read from a download body between progress reports.
40const READ_CHUNK: usize = 64 * 1024;
41
42/// Times a dropped connection is retried before the transfer gives up.
43const MAX_ATTEMPTS: u32 = 5;
44
45/// Which way a resumable transfer moves data.
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum TransferDirection {
48    /// Downloading a bundle from the server.
49    Download,
50    /// Uploading a bundle to the server.
51    Upload,
52}
53
54/// Progress of a resumable transfer, reported through
55/// [`Progress::resumable_transfer`].
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum ResumableEvent {
58    /// The transfer began. `resumed_from` is how many bytes an earlier attempt
59    /// had already moved (zero for a fresh transfer).
60    Started {
61        /// Download or upload.
62        direction: TransferDirection,
63        /// The URL being transferred.
64        url: String,
65        /// Total bytes, when known.
66        total: Option<u64>,
67        /// Bytes already in place before this run.
68        resumed_from: u64,
69    },
70    /// More bytes arrived or were accepted.
71    Advanced {
72        /// Bytes in place so far, including `resumed_from`.
73        done: u64,
74        /// Total bytes, when known.
75        total: Option<u64>,
76    },
77    /// The connection dropped; the transfer is picking up again at `done`.
78    Reconnecting {
79        /// Bytes in place when the connection dropped.
80        done: u64,
81        /// Which retry this is, starting at 1.
82        attempt: u32,
83    },
84    /// Every byte is in place.
85    Finished {
86        /// Total bytes transferred.
87        done: u64,
88        /// Bytes that were already in place before this run.
89        resumed_from: u64,
90    },
91}
92
93fn resumable_status_message(url: &str, status: u16) -> String {
94    format!("HTTP {status} from {}", redact_url_for_display(url))
95}
96
97fn resumable_protocol_message(url: &str, detail: &str) -> String {
98    format!(
99        "unexpected response from {}: {detail}",
100        redact_url_for_display(url)
101    )
102}
103
104fn resumable_exhausted_message(url: &str, attempts: u32, last: &str) -> String {
105    format!(
106        "the transfer of {} kept failing after {attempts} attempts: {last}",
107        redact_url_for_display(url)
108    )
109}
110
111/// Why a resumable transfer failed.
112#[derive(Debug, thiserror::Error)]
113#[non_exhaustive]
114pub enum ResumableError {
115    /// The server answered with a status the transfer cannot continue from.
116    #[error("{}", resumable_status_message(.url, *status))]
117    Status {
118        /// The request URL.
119        url: String,
120        /// The response status.
121        status: u16,
122    },
123    /// The server's answer did not follow the protocol.
124    #[error("{}", resumable_protocol_message(.url, .detail))]
125    Protocol {
126        /// The request URL.
127        url: String,
128        /// What was wrong.
129        detail: String,
130    },
131    /// The connection kept dropping.
132    #[error("{}", resumable_exhausted_message(.url, *attempts, .last))]
133    Exhausted {
134        /// The URL being transferred.
135        url: String,
136        /// Attempts made.
137        attempts: u32,
138        /// The last failure.
139        last: String,
140    },
141}
142
143impl From<ResumableError> for crate::error::Error {
144    fn from(err: ResumableError) -> Self {
145        Self::Remote(crate::remote::RemoteError::Resumable(Box::new(err)))
146    }
147}
148
149use crate::error::Result;
150
151/// What [`download_resumable`] did.
152#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153pub struct DownloadOutcome {
154    /// Size of the downloaded file.
155    pub bytes: u64,
156    /// Bytes that were already on disk from an earlier, interrupted run.
157    pub resumed_from: u64,
158}
159
160/// What is known about a partial download, kept next to it as JSON.
161#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
162struct DownloadState {
163    /// The URL the bytes came from; another URL starts over.
164    url: String,
165    /// The server's `ETag` (or `Last-Modified`), sent back as `If-Range` so a
166    /// changed file is downloaded afresh instead of spliced.
167    validator: Option<String>,
168    /// Total size, once the server said.
169    total: Option<u64>,
170    /// Whether every byte is in place (the file is at its final path).
171    complete: bool,
172}
173
174/// The files a download to `dest` keeps: the bytes so far and their state.
175fn download_paths(dest: &Path) -> (PathBuf, PathBuf) {
176    let name = dest
177        .file_name()
178        .map(|n| n.to_string_lossy().into_owned())
179        .unwrap_or_default();
180    (
181        dest.with_file_name(format!("{name}.part")),
182        dest.with_file_name(format!("{name}.json")),
183    )
184}
185
186fn load_state(path: &Path) -> Option<DownloadState> {
187    serde_json::from_slice(&std::fs::read(path).ok()?).ok()
188}
189
190fn save_state(path: &Path, state: &DownloadState) -> Result<()> {
191    let tmp = path.with_extension("json.tmp");
192    let bytes = serde_json::to_vec(state).map_err(|e| std::io::Error::other(e.to_string()))?;
193    std::fs::write(&tmp, bytes)?;
194    std::fs::rename(&tmp, path)?;
195    Ok(())
196}
197
198/// Where to download when several processes share one resumable cache slot.
199#[derive(Debug)]
200pub enum ResumableDownloadDest {
201    /// The shared `dest`; hold `lock` until [`discard_download`] on it finishes.
202    Shared {
203        /// Path passed to [`download_resumable`].
204        dest: PathBuf,
205        /// Exclusive lock on the sibling `.lock` file for `dest`.
206        lock: File,
207    },
208    /// A one-off path in the cache directory; safe to use without locking.
209    Private(PathBuf),
210}
211
212/// Pick a resumable download path under a shared bundle cache.
213///
214/// Creates `cache_dir` when missing, then tries an exclusive lock on a sibling
215/// `.lock` file next to `shared_dest`. While that lock is held, only this
216/// process may use `shared_dest` and its `.part`/`.json` sidecars. When another
217/// process already holds the lock, returns a unique private path in `cache_dir`
218/// so parallel clones do not clobber the shared partial download.
219///
220/// # Errors
221///
222/// I/O errors opening or locking the lock file, or creating `cache_dir`.
223pub fn claim_resumable_download(
224    cache_dir: &Path,
225    shared_dest: &Path,
226) -> std::io::Result<ResumableDownloadDest> {
227    std::fs::create_dir_all(cache_dir)?;
228    let lock_path = shared_download_lock_path(shared_dest);
229    let file = OpenOptions::new()
230        .create(true)
231        .read(true)
232        .write(true)
233        .truncate(false)
234        .open(&lock_path)?;
235    match file.try_lock() {
236        Ok(()) => Ok(ResumableDownloadDest::Shared {
237            dest: shared_dest.to_path_buf(),
238            lock: file,
239        }),
240        Err(std::fs::TryLockError::WouldBlock) => Ok(ResumableDownloadDest::Private(
241            private_resumable_download_dest(cache_dir, shared_dest),
242        )),
243        Err(std::fs::TryLockError::Error(e)) => Err(e),
244    }
245}
246
247fn shared_download_lock_path(dest: &Path) -> PathBuf {
248    dest.with_extension("lock")
249}
250
251fn private_resumable_download_dest(cache_dir: &Path, shared_dest: &Path) -> PathBuf {
252    use std::sync::atomic::{AtomicU64, Ordering};
253    static SUFFIX: AtomicU64 = AtomicU64::new(0);
254    let stem = shared_dest
255        .file_stem()
256        .and_then(|s| s.to_str())
257        .unwrap_or("bundle");
258    let n = SUFFIX.fetch_add(1, Ordering::Relaxed);
259    cache_dir.join(format!("{stem}.{}.{n}.private.bundle", std::process::id()))
260}
261
262/// Remove `dest` and everything [`download_resumable`] keeps for it.
263///
264/// # Errors
265///
266/// A removal failure other than the file not existing.
267pub fn discard_download(dest: &Path) -> std::io::Result<()> {
268    let (part, state) = download_paths(dest);
269    for path in [dest, part.as_path(), state.as_path()] {
270        match std::fs::remove_file(path) {
271            Ok(()) => {}
272            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
273            Err(e) => return Err(e),
274        }
275    }
276    Ok(())
277}
278
279/// Download `url` to `dest`, continuing an earlier interrupted download.
280///
281/// Bytes arrive in `<dest>.part`, with what is known about them in
282/// `<dest>.json`; the file is renamed to `dest` once complete. When a
283/// previous run left a partial file for the same URL, only the rest is asked
284/// for (`Range: bytes=<n>-`, with `If-Range` so a file that changed on the
285/// server starts over). A connection that drops mid-download is retried a few
286/// times from where it stopped. When `dest` is already complete for `url`,
287/// nothing is downloaded.
288///
289/// # Parameters
290///
291/// - `client`: sends the requests ([`HttpClient::send`]).
292/// - `url`: what to download.
293/// - `dest`: where the finished file goes; its directory must exist. When
294///   several processes share a cache directory, callers must use
295///   [`claim_resumable_download`] first so only one process uses a given
296///   `dest` at a time.
297/// - `progress`: receives [`ResumableEvent`]s.
298/// - `cancel`: checked before each request and between body chunks.
299///
300/// # Errors
301///
302/// [`ResumableError::Status`] when the server refuses the download,
303/// [`ResumableError::Exhausted`] when the connection keeps dropping,
304/// [`crate::error::Error::Cancelled`] when `cancel` fires, and I/O errors from the
305/// local files. The partial file is kept so a later call can resume.
306pub fn download_resumable(
307    client: &dyn HttpClient,
308    url: &str,
309    dest: &Path,
310    progress: &mut dyn Progress,
311    cancel: &CancelToken,
312) -> Result<DownloadOutcome> {
313    let (part, state_path) = download_paths(dest);
314    let mut state = load_state(&state_path).unwrap_or_default();
315    if state.url != url {
316        discard_download(dest)?;
317        state = DownloadState {
318            url: url.to_owned(),
319            ..DownloadState::default()
320        };
321    }
322    if state.complete {
323        if let (Some(total), Ok(meta)) = (state.total, std::fs::metadata(dest)) {
324            if meta.len() == total {
325                return Ok(DownloadOutcome {
326                    bytes: total,
327                    resumed_from: total,
328                });
329            }
330        }
331        discard_download(dest)?;
332        state = DownloadState {
333            url: url.to_owned(),
334            ..DownloadState::default()
335        };
336    }
337
338    let resumed_from = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
339    let mut started = false;
340    let mut attempt = 0;
341    let mut last_failure = String::new();
342    loop {
343        if cancel.is_cancelled() {
344            return Err(crate::error::Error::Cancelled);
345        }
346        if attempt > 0 {
347            let done = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
348            if attempt >= MAX_ATTEMPTS {
349                return Err(ResumableError::Exhausted {
350                    url: url.to_owned(),
351                    attempts: attempt,
352                    last: last_failure,
353                }
354                .into());
355            }
356            progress.resumable_transfer(&ResumableEvent::Reconnecting { done, attempt });
357        }
358        attempt += 1;
359        match download_attempt(
360            client,
361            url,
362            &part,
363            &mut state,
364            &state_path,
365            progress,
366            cancel,
367            |p, s| {
368                if !started {
369                    started = true;
370                    p.start_with_unit(
371                        ProgressPhase::TransferringBundle,
372                        ProgressUnit::Bytes,
373                        s.total,
374                    );
375                    p.set(s.resumed_from);
376                    p.resumable_transfer(&ResumableEvent::Started {
377                        direction: TransferDirection::Download,
378                        url: url.to_owned(),
379                        total: s.total,
380                        resumed_from: s.resumed_from,
381                    });
382                }
383            },
384        ) {
385            Ok(AttemptOutcome::Complete(bytes)) => {
386                std::fs::rename(&part, dest)?;
387                state.complete = true;
388                state.total = Some(bytes);
389                save_state(&state_path, &state)?;
390                progress.set(bytes);
391                progress.finish();
392                progress.resumable_transfer(&ResumableEvent::Finished {
393                    done: bytes,
394                    resumed_from: resumed_from.min(bytes),
395                });
396                return Ok(DownloadOutcome {
397                    bytes,
398                    resumed_from: resumed_from.min(bytes),
399                });
400            }
401            Ok(AttemptOutcome::Dropped(why)) => last_failure = why,
402            Err(e) => return Err(e),
403        }
404    }
405}
406
407/// Where a download attempt starts, reported once when the first one answers.
408struct AttemptStart {
409    total: Option<u64>,
410    resumed_from: u64,
411}
412
413enum AttemptOutcome {
414    /// Every byte is in the partial file; the value is its size.
415    Complete(u64),
416    /// The connection dropped before the end.
417    Dropped(String),
418}
419
420/// One request of a download: ask for the bytes not yet on disk and append
421/// them to `part`.
422#[allow(clippy::too_many_arguments)]
423fn download_attempt(
424    client: &dyn HttpClient,
425    url: &str,
426    part: &Path,
427    state: &mut DownloadState,
428    state_path: &Path,
429    progress: &mut dyn Progress,
430    cancel: &CancelToken,
431    mut on_start: impl FnMut(&mut dyn Progress, &AttemptStart),
432) -> Result<AttemptOutcome> {
433    let have = std::fs::metadata(part).map(|m| m.len()).unwrap_or(0);
434    let mut request = HttpRequest::new(HttpMethod::Get, url).long_running();
435    if have > 0 {
436        request = request.header("Range", format!("bytes={have}-"));
437        if let Some(validator) = &state.validator {
438            request = request.header("If-Range", validator.clone());
439        }
440    }
441    let response = match client.send(&request) {
442        Ok(r) => r,
443        Err(e)
444            if matches!(
445                e,
446                crate::error::Error::Remote(crate::remote::RemoteError::Auth { .. })
447            ) =>
448        {
449            return Err(e);
450        }
451        Err(e) => return Ok(AttemptOutcome::Dropped(e.to_string())),
452    };
453    let (mut offset, total) =
454        match response.status {
455            206 if have > 0 => {
456                let (start, total) = parse_content_range(response.header("content-range"))
457                    .ok_or_else(|| ResumableError::Protocol {
458                        url: url.to_owned(),
459                        detail: "206 without a usable Content-Range".to_owned(),
460                    })?;
461                if start != have {
462                    return Err(ResumableError::Protocol {
463                        url: url.to_owned(),
464                        detail: format!("asked for bytes from {have}, got bytes from {start}"),
465                    }
466                    .into());
467                }
468                (have, total)
469            }
470            200 | 206 => {
471                // A full answer: the server ignored the range or the file changed.
472                let total = response
473                    .header("content-length")
474                    .and_then(|v| v.trim().parse().ok());
475                (0, total)
476            }
477            416 if have > 0 && state.total == Some(have) => {
478                on_start(
479                    progress,
480                    &AttemptStart {
481                        total: state.total,
482                        resumed_from: have,
483                    },
484                );
485                return Ok(AttemptOutcome::Complete(have));
486            }
487            416 => {
488                // The partial file is longer than the server's: start over.
489                File::create(part)?;
490                return Ok(AttemptOutcome::Dropped(
491                    "the server's file is shorter than the partial download".to_owned(),
492                ));
493            }
494            status => {
495                return Err(ResumableError::Status {
496                    url: url.to_owned(),
497                    status,
498                }
499                .into())
500            }
501        };
502    state.total = total;
503    state.validator = validator(&response);
504    save_state(state_path, state)?;
505    on_start(
506        progress,
507        &AttemptStart {
508            total,
509            resumed_from: offset,
510        },
511    );
512
513    let mut file = if offset == 0 {
514        File::create(part)?
515    } else {
516        OpenOptions::new().append(true).open(part)?
517    };
518    let mut body = response.body;
519    let mut buf = vec![0u8; READ_CHUNK];
520    loop {
521        if cancel.is_cancelled() {
522            file.flush()?;
523            return Err(crate::error::Error::Cancelled);
524        }
525        let n = match body.read(&mut buf) {
526            Ok(0) => break,
527            Ok(n) => n,
528            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
529            Err(e) => {
530                file.flush()?;
531                return Ok(AttemptOutcome::Dropped(e.to_string()));
532            }
533        };
534        file.write_all(&buf[..n])?;
535        offset += n as u64;
536        progress.set(offset);
537        progress.resumable_transfer(&ResumableEvent::Advanced {
538            done: offset,
539            total,
540        });
541    }
542    file.flush()?;
543    file.sync_all()?;
544    match total {
545        Some(total) if offset < total => Ok(AttemptOutcome::Dropped(format!(
546            "the connection closed after {offset} of {total} bytes"
547        ))),
548        _ => Ok(AttemptOutcome::Complete(offset)),
549    }
550}
551
552/// The value to send back as `If-Range`: a strong `ETag`, else `Last-Modified`.
553fn validator(response: &HttpResponse) -> Option<String> {
554    response
555        .header("etag")
556        .filter(|v| !v.starts_with("W/"))
557        .or_else(|| response.header("last-modified"))
558        .map(str::to_owned)
559}
560
561/// Parse `bytes <start>-<end>/<total>` into the start and (known) total.
562fn parse_content_range(value: Option<&str>) -> Option<(u64, Option<u64>)> {
563    let rest = value?.trim().strip_prefix("bytes ")?;
564    let (range, total) = rest.split_once('/')?;
565    let (start, _end) = range.split_once('-')?;
566    let start = start.trim().parse().ok()?;
567    let total = match total.trim() {
568        "*" => None,
569        t => Some(t.parse().ok()?),
570    };
571    Some((start, total))
572}
573
574/// An upload to a TUS endpoint.
575///
576/// [`TusUpload::create`] asks the endpoint for a new upload of a known length
577/// and returns its URL; [`TusUpload::send_file`] sends a file to it, starting
578/// at whatever offset the server already has, so calling it again after an
579/// interruption (with the same upload URL) only sends the rest.
580#[derive(Debug, Clone, PartialEq, Eq)]
581pub(crate) struct TusUpload {
582    /// Absolute URL of the upload.
583    pub url: String,
584    /// Total bytes of the upload.
585    pub length: u64,
586}
587
588impl TusUpload {
589    /// Create an upload of `length` bytes at `endpoint` (TUS creation
590    /// extension).
591    ///
592    /// # Errors
593    ///
594    /// [`ResumableError::Status`] when the server refuses, and
595    /// [`ResumableError::Protocol`] when it answers without a `Location`.
596    pub fn create(client: &dyn HttpClient, endpoint: &str, length: u64) -> Result<Self> {
597        let request = HttpRequest::new(HttpMethod::Post, endpoint)
598            .header("Tus-Resumable", TUS_VERSION)
599            .header("Upload-Length", length.to_string());
600        let response = client.send(&request)?;
601        if response.status != 201 {
602            return Err(ResumableError::Status {
603                url: endpoint.to_owned(),
604                status: response.status,
605            }
606            .into());
607        }
608        let location = response
609            .header("location")
610            .ok_or_else(|| ResumableError::Protocol {
611                url: endpoint.to_owned(),
612                detail: "no Location for the new upload".to_owned(),
613            })?;
614        let url = resolve_url(endpoint, location).ok_or_else(|| ResumableError::Protocol {
615            url: endpoint.to_owned(),
616            detail: format!("bad Location {location:?}"),
617        })?;
618        Ok(Self { url, length })
619    }
620
621    /// How many bytes the server has, or `None` when it no longer knows the
622    /// upload (expired or already used).
623    ///
624    /// # Errors
625    ///
626    /// [`ResumableError::Status`] for other refusals, and
627    /// [`ResumableError::Protocol`] for an answer without `Upload-Offset`.
628    pub fn offset(&self, client: &dyn HttpClient) -> Result<Option<u64>> {
629        let request =
630            HttpRequest::new(HttpMethod::Head, &self.url).header("Tus-Resumable", TUS_VERSION);
631        let response = client.send(&request)?;
632        match response.status {
633            404 | 410 => Ok(None),
634            s if (200..300).contains(&s) => upload_offset(&response, &self.url).map(Some),
635            status => Err(ResumableError::Status {
636                url: self.url.clone(),
637                status,
638            }
639            .into()),
640        }
641    }
642
643    /// Send `file` (exactly [`Self::length`] bytes) to the upload, starting at
644    /// the server's offset and resynchronizing after each failed request.
645    ///
646    /// # Errors
647    ///
648    /// [`ResumableError::Status`] when the server no longer knows the upload
649    /// (`404`/`410`) or refuses it, [`ResumableError::Exhausted`] when requests
650    /// keep failing, and I/O errors reading `file`.
651    pub fn send_file(
652        &self,
653        client: &dyn HttpClient,
654        file: &Path,
655        progress: &mut dyn Progress,
656    ) -> Result<()> {
657        let gone = |status| ResumableError::Status {
658            url: self.url.clone(),
659            status,
660        };
661        let mut offset = self.offset(client)?.ok_or_else(|| gone(404))?;
662        let resumed_from = offset;
663        progress.start_with_unit(
664            ProgressPhase::TransferringBundle,
665            ProgressUnit::Bytes,
666            Some(self.length),
667        );
668        progress.set(offset);
669        progress.resumable_transfer(&ResumableEvent::Started {
670            direction: TransferDirection::Upload,
671            url: self.url.clone(),
672            total: Some(self.length),
673            resumed_from,
674        });
675        let mut input = File::open(file)?;
676        let mut buf = vec![0u8; PATCH_CHUNK];
677        let mut failures = 0;
678        while offset < self.length {
679            let n = usize::try_from(self.length - offset)
680                .map_or(PATCH_CHUNK, |left| left.min(PATCH_CHUNK));
681            input.seek(SeekFrom::Start(offset))?;
682            input.read_exact(&mut buf[..n])?;
683            let request = HttpRequest::new(HttpMethod::Patch, &self.url)
684                .header("Tus-Resumable", TUS_VERSION)
685                .header("Upload-Offset", offset.to_string())
686                .header("Content-Type", TUS_PATCH_CONTENT_TYPE)
687                .body(&buf[..n])
688                .long_running();
689            let failure = match client.send(&request) {
690                Ok(r) if r.status == 204 || r.status == 200 => {
691                    offset = upload_offset(&r, &self.url)?;
692                    failures = 0;
693                    progress.set(offset);
694                    progress.resumable_transfer(&ResumableEvent::Advanced {
695                        done: offset,
696                        total: Some(self.length),
697                    });
698                    continue;
699                }
700                Ok(r) if r.status == 404 || r.status == 410 => {
701                    return Err(crate::error::Error::from(gone(r.status)))
702                }
703                Ok(r) if r.status == 409 => format!("offset {offset} was out of date"),
704                Ok(r) if (400..500).contains(&r.status) => {
705                    return Err(crate::error::Error::from(gone(r.status)))
706                }
707                Ok(r) => format!("HTTP {}", r.status),
708                Err(e)
709                    if matches!(
710                        e,
711                        crate::error::Error::Remote(crate::remote::RemoteError::Auth { .. })
712                    ) =>
713                {
714                    return Err(e);
715                }
716                Err(e) => e.to_string(),
717            };
718            failures += 1;
719            if failures >= MAX_ATTEMPTS {
720                return Err(ResumableError::Exhausted {
721                    url: self.url.clone(),
722                    attempts: failures,
723                    last: failure,
724                }
725                .into());
726            }
727            progress.resumable_transfer(&ResumableEvent::Reconnecting {
728                done: offset,
729                attempt: failures,
730            });
731            // Ask the server where it got to; it may have kept part of the chunk.
732            if let Ok(Some(server)) = self.offset(client) {
733                offset = server;
734            }
735        }
736        progress.set(self.length);
737        progress.finish();
738        progress.resumable_transfer(&ResumableEvent::Finished {
739            done: self.length,
740            resumed_from,
741        });
742        Ok(())
743    }
744}
745
746fn upload_offset(response: &HttpResponse, url: &str) -> Result<u64> {
747    Ok(response
748        .header("upload-offset")
749        .and_then(|v| v.trim().parse().ok())
750        .ok_or_else(|| ResumableError::Protocol {
751            url: url.to_owned(),
752            detail: "no Upload-Offset".to_owned(),
753        })?)
754}
755
756/// Resolve `reference` (absolute, or relative to `base`) to an absolute URL.
757#[must_use]
758pub(crate) fn resolve_url(base: &str, reference: &str) -> Option<String> {
759    if let Ok(absolute) = url::Url::parse(reference) {
760        return Some(absolute.to_string());
761    }
762    url::Url::parse(base)
763        .ok()?
764        .join(reference)
765        .ok()
766        .map(|u| u.to_string())
767}
768
769#[cfg(test)]
770mod tests {
771    use super::*;
772    use crate::fetch::NoProgress;
773    use crate::progress::CancelToken;
774    use crate::remote::RemoteError;
775    use std::collections::VecDeque;
776    use std::sync::Mutex;
777
778    /// Serves one file from memory, honoring `Range`, and can cut a body short.
779    struct FileServer {
780        data: Vec<u8>,
781        etag: String,
782        /// Bytes to send before dropping, one entry per GET (then full bodies).
783        cuts: Mutex<VecDeque<usize>>,
784        ranges: Mutex<Vec<Option<String>>>,
785    }
786
787    struct Cut {
788        data: std::io::Cursor<Vec<u8>>,
789        fail_after: bool,
790    }
791
792    impl Read for Cut {
793        fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
794            let n = self.data.read(buf)?;
795            if n == 0 && self.fail_after {
796                return Err(std::io::Error::new(
797                    std::io::ErrorKind::ConnectionReset,
798                    "reset",
799                ));
800            }
801            Ok(n)
802        }
803    }
804
805    impl HttpClient for FileServer {
806        fn get(&self, _: &str, _: Option<&str>) -> crate::error::Result<Vec<u8>> {
807            unreachable!()
808        }
809        fn post(
810            &self,
811            _: &str,
812            _: &str,
813            _: &str,
814            _: &[u8],
815            _: Option<&str>,
816        ) -> crate::error::Result<Vec<u8>> {
817            unreachable!()
818        }
819        fn send(&self, request: &HttpRequest<'_>) -> crate::error::Result<HttpResponse> {
820            let range = request
821                .headers
822                .iter()
823                .find(|(n, _)| *n == "Range")
824                .map(|(_, v)| v.clone());
825            let if_range = request
826                .headers
827                .iter()
828                .find(|(n, _)| *n == "If-Range")
829                .map(|(_, v)| v.clone());
830            self.ranges
831                .lock()
832                .unwrap_or_else(|e| e.into_inner())
833                .push(range.clone());
834            let start = range
835                .as_deref()
836                .and_then(|r| r.strip_prefix("bytes="))
837                .and_then(|r| r.strip_suffix('-'))
838                .and_then(|r| r.parse::<usize>().ok())
839                .filter(|_| if_range.as_deref().is_none_or(|v| v == self.etag));
840            let total = self.data.len();
841            let (status, mut headers, body) = match start {
842                Some(s) if s >= total => (416, vec![], vec![]),
843                Some(s) => (
844                    206,
845                    vec![(
846                        "content-range".to_owned(),
847                        format!("bytes {s}-{}/{total}", total - 1),
848                    )],
849                    self.data[s..].to_vec(),
850                ),
851                None => (
852                    200,
853                    vec![("content-length".to_owned(), total.to_string())],
854                    self.data.clone(),
855                ),
856            };
857            headers.push(("etag".to_owned(), self.etag.clone()));
858            let cut = self
859                .cuts
860                .lock()
861                .unwrap_or_else(|e| e.into_inner())
862                .pop_front();
863            let (body, fail_after) = match cut {
864                Some(n) if n < body.len() => (body[..n].to_vec(), true),
865                _ => (body, false),
866            };
867            Ok(HttpResponse {
868                status,
869                headers,
870                body: Box::new(Cut {
871                    data: std::io::Cursor::new(body),
872                    fail_after,
873                }),
874            })
875        }
876    }
877
878    #[derive(Default)]
879    struct Events(Vec<ResumableEvent>);
880    impl crate::progress::ProgressSink for Events {}
881    impl Progress for Events {
882        fn resumable_transfer(&mut self, event: &ResumableEvent) {
883            if !matches!(event, ResumableEvent::Advanced { .. }) {
884                self.0.push(event.clone());
885            }
886        }
887    }
888
889    fn server(cuts: &[usize]) -> FileServer {
890        FileServer {
891            data: (0..200_000u32).map(|i| (i % 251) as u8).collect(),
892            etag: "\"v1\"".to_owned(),
893            cuts: Mutex::new(cuts.iter().copied().collect()),
894            ranges: Mutex::new(Vec::new()),
895        }
896    }
897
898    fn tempdir() -> tempfile::TempDir {
899        match tempfile::tempdir() {
900            Ok(d) => d,
901            Err(e) => panic!("tempdir: {e}"),
902        }
903    }
904
905    #[test]
906    fn dropped_downloads_resume_in_the_same_run() {
907        let dir = tempdir();
908        let dest = dir.path().join("clone.bundle");
909        let srv = server(&[70_000, 10_000]);
910        let mut events = Events::default();
911        let outcome = download_resumable(
912            &srv,
913            "http://h/b",
914            &dest,
915            &mut events,
916            &CancelToken::never(),
917        );
918        assert_eq!(
919            outcome.ok(),
920            Some(DownloadOutcome {
921                bytes: 200_000,
922                resumed_from: 0
923            })
924        );
925        assert_eq!(std::fs::read(&dest).ok(), Some(srv.data.clone()));
926        let ranges = srv.ranges.lock().unwrap_or_else(|e| e.into_inner()).clone();
927        assert_eq!(
928            ranges,
929            [
930                None,
931                Some("bytes=70000-".to_owned()),
932                Some("bytes=80000-".to_owned())
933            ]
934        );
935        assert!(matches!(
936            events.0.as_slice(),
937            [
938                ResumableEvent::Started {
939                    resumed_from: 0,
940                    total: Some(200_000),
941                    ..
942                },
943                ResumableEvent::Reconnecting {
944                    done: 70_000,
945                    attempt: 1
946                },
947                ResumableEvent::Reconnecting {
948                    done: 80_000,
949                    attempt: 2
950                },
951                ResumableEvent::Finished {
952                    done: 200_000,
953                    resumed_from: 0
954                },
955            ]
956        ));
957    }
958
959    #[test]
960    fn a_later_run_resumes_a_partial_download() {
961        let dir = tempdir();
962        let dest = dir.path().join("clone.bundle");
963        // The first run's connection keeps dropping until it gives up.
964        let failing = server(&[50_000, 0, 0, 0, 0, 0]);
965        assert!(matches!(
966            download_resumable(
967                &failing,
968                "http://h/b",
969                &dest,
970                &mut NoProgress,
971                &CancelToken::never()
972            ),
973            Err(crate::error::Error::Remote(crate::remote::RemoteError::Resumable(ref r)))
974                if matches!(**r, ResumableError::Exhausted { .. })
975        ));
976        assert!(!dest.exists());
977
978        let srv = server(&[]);
979        let mut events = Events::default();
980        let outcome = download_resumable(
981            &srv,
982            "http://h/b",
983            &dest,
984            &mut events,
985            &CancelToken::never(),
986        );
987        assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(50_000));
988        assert_eq!(std::fs::read(&dest).ok(), Some(srv.data.clone()));
989        assert!(matches!(
990            events.0.first(),
991            Some(ResumableEvent::Started {
992                resumed_from: 50_000,
993                ..
994            })
995        ));
996
997        // Complete already: nothing is requested.
998        let again = server(&[]);
999        let outcome = download_resumable(
1000            &again,
1001            "http://h/b",
1002            &dest,
1003            &mut NoProgress,
1004            &CancelToken::never(),
1005        );
1006        assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(200_000));
1007        assert!(again
1008            .ranges
1009            .lock()
1010            .unwrap_or_else(|e| e.into_inner())
1011            .is_empty());
1012    }
1013
1014    #[test]
1015    fn a_changed_file_or_url_starts_over() {
1016        let dir = tempdir();
1017        let dest = dir.path().join("clone.bundle");
1018        let failing = server(&[50_000, 0, 0, 0, 0, 0]);
1019        let _ = download_resumable(
1020            &failing,
1021            "http://h/b",
1022            &dest,
1023            &mut NoProgress,
1024            &CancelToken::never(),
1025        );
1026
1027        let mut changed = server(&[]);
1028        changed.etag = "\"v2\"".to_owned();
1029        let outcome = download_resumable(
1030            &changed,
1031            "http://h/b",
1032            &dest,
1033            &mut NoProgress,
1034            &CancelToken::never(),
1035        );
1036        assert_eq!(outcome.ok().map(|o| o.bytes), Some(200_000));
1037        assert_eq!(std::fs::read(&dest).ok(), Some(changed.data.clone()));
1038
1039        let other = server(&[]);
1040        let outcome = download_resumable(
1041            &other,
1042            "http://h/other",
1043            &dest,
1044            &mut NoProgress,
1045            &CancelToken::never(),
1046        );
1047        assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(0));
1048        assert_eq!(
1049            other
1050                .ranges
1051                .lock()
1052                .unwrap_or_else(|e| e.into_inner())
1053                .as_slice(),
1054            [None]
1055        );
1056    }
1057
1058    /// Cancels once `after` bytes of the download are on disk.
1059    struct CancelAfter {
1060        flag: CancelToken,
1061        after: u64,
1062    }
1063    impl crate::progress::ProgressSink for CancelAfter {}
1064    impl Progress for CancelAfter {
1065        fn resumable_transfer(&mut self, event: &ResumableEvent) {
1066            if let ResumableEvent::Advanced { done, .. } = event {
1067                if *done >= self.after {
1068                    self.flag.cancel();
1069                }
1070            }
1071        }
1072    }
1073
1074    #[test]
1075    fn a_cancelled_download_stops_mid_body_and_resumes_next_time() {
1076        let dir = tempdir();
1077        let dest = dir.path().join("clone.bundle");
1078        let srv = server(&[]);
1079        let flag = CancelToken::new();
1080        let mut progress = CancelAfter {
1081            flag: flag.clone(),
1082            after: 1,
1083        };
1084        assert!(matches!(
1085            download_resumable(&srv, "http://h/b", &dest, &mut progress, &flag),
1086            Err(crate::error::Error::Cancelled)
1087        ));
1088        assert!(!dest.exists());
1089        let (part, _) = download_paths(&dest);
1090        let kept = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
1091        assert!(
1092            kept > 0 && kept < 200_000,
1093            "one chunk is kept, got {kept} bytes"
1094        );
1095
1096        let outcome = download_resumable(
1097            &srv,
1098            "http://h/b",
1099            &dest,
1100            &mut NoProgress,
1101            &CancelToken::never(),
1102        );
1103        assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(kept));
1104        assert_eq!(std::fs::read(&dest).ok(), Some(srv.data.clone()));
1105    }
1106
1107    #[test]
1108    fn claim_resumable_download_serializes_the_shared_slot() {
1109        let dir = tempdir();
1110        let shared = dir.path().join("abc.bundle");
1111        let first = claim_resumable_download(dir.path(), &shared).expect("first claim");
1112        let (first_dest, first_lock) = match first {
1113            ResumableDownloadDest::Shared { dest, lock } => (dest, lock),
1114            ResumableDownloadDest::Private(_) => panic!("first claim should be shared"),
1115        };
1116        assert_eq!(first_dest, shared);
1117        let second = claim_resumable_download(dir.path(), &shared).expect("second claim");
1118        assert!(
1119            matches!(second, ResumableDownloadDest::Private(_)),
1120            "second claim must not touch the shared partial files"
1121        );
1122        let other = std::fs::File::open(shared_download_lock_path(&shared)).expect("lock file");
1123        assert!(
1124            matches!(other.try_lock(), Err(std::fs::TryLockError::WouldBlock)),
1125            "lock file stays held until the first guard drops"
1126        );
1127        drop(first_lock);
1128    }
1129
1130    #[test]
1131    fn auth_failure_does_not_retry_the_download() {
1132        use crate::error::AuthFailure;
1133
1134        struct AuthClient;
1135        impl HttpClient for AuthClient {
1136            fn get(&self, _: &str, _: Option<&str>) -> crate::error::Result<Vec<u8>> {
1137                unreachable!()
1138            }
1139            fn post(
1140                &self,
1141                _: &str,
1142                _: &str,
1143                _: &str,
1144                _: &[u8],
1145                _: Option<&str>,
1146            ) -> crate::error::Result<Vec<u8>> {
1147                unreachable!()
1148            }
1149            fn send(&self, _: &HttpRequest<'_>) -> crate::error::Result<HttpResponse> {
1150                Err(RemoteError::Auth {
1151                    url: "http://h/b".to_owned(),
1152                    reason: AuthFailure::Rejected,
1153                }
1154                .into())
1155            }
1156        }
1157
1158        let dir = tempdir();
1159        let dest = dir.path().join("clone.bundle");
1160        let outcome = download_resumable(
1161            &AuthClient,
1162            "http://h/b",
1163            &dest,
1164            &mut NoProgress,
1165            &CancelToken::never(),
1166        );
1167        assert!(
1168            matches!(
1169                outcome,
1170                Err(crate::error::Error::Remote(
1171                    crate::remote::RemoteError::Auth { .. }
1172                ))
1173            ),
1174            "auth errors must fail immediately: {outcome:?}"
1175        );
1176        assert!(
1177            !matches!(
1178                &outcome,
1179                Err(crate::error::Error::Remote(crate::remote::RemoteError::Resumable(r)))
1180                    if matches!(**r, ResumableError::Exhausted { .. })
1181            ),
1182            "auth must not be retried like a dropped connection"
1183        );
1184    }
1185
1186    #[test]
1187    fn an_already_cancelled_download_sends_no_request() {
1188        let dir = tempdir();
1189        let dest = dir.path().join("clone.bundle");
1190        let srv = server(&[]);
1191        let flag = CancelToken::new();
1192        flag.cancel();
1193        assert!(matches!(
1194            download_resumable(&srv, "http://h/b", &dest, &mut NoProgress, &flag),
1195            Err(crate::error::Error::Cancelled)
1196        ));
1197        assert!(srv
1198            .ranges
1199            .lock()
1200            .unwrap_or_else(|e| e.into_inner())
1201            .is_empty());
1202    }
1203
1204    #[test]
1205    fn content_ranges_parse() {
1206        assert_eq!(
1207            parse_content_range(Some("bytes 10-99/100")),
1208            Some((10, Some(100)))
1209        );
1210        assert_eq!(parse_content_range(Some("bytes 10-99/*")), Some((10, None)));
1211        assert_eq!(parse_content_range(Some("items 1-2/3")), None);
1212        assert_eq!(parse_content_range(None), None);
1213    }
1214
1215    #[test]
1216    fn urls_resolve_against_the_endpoint() {
1217        assert_eq!(
1218            resolve_url("http://h:1/repo/push", "/repo/push/abc").as_deref(),
1219            Some("http://h:1/repo/push/abc")
1220        );
1221        assert_eq!(
1222            resolve_url("http://h/repo/push", "https://other/x").as_deref(),
1223            Some("https://other/x")
1224        );
1225    }
1226
1227    /// A TUS server in memory that drops some PATCHes after keeping part of them.
1228    struct TusServer {
1229        stored: Mutex<Vec<u8>>,
1230        /// Per PATCH: bytes to keep before failing (`None` = accept all).
1231        patch_cuts: Mutex<VecDeque<Option<usize>>>,
1232        length: Mutex<u64>,
1233    }
1234
1235    impl HttpClient for TusServer {
1236        fn get(&self, _: &str, _: Option<&str>) -> crate::error::Result<Vec<u8>> {
1237            unreachable!()
1238        }
1239        fn post(
1240            &self,
1241            _: &str,
1242            _: &str,
1243            _: &str,
1244            _: &[u8],
1245            _: Option<&str>,
1246        ) -> crate::error::Result<Vec<u8>> {
1247            unreachable!()
1248        }
1249        fn send(&self, request: &HttpRequest<'_>) -> crate::error::Result<HttpResponse> {
1250            let header = |name: &str| {
1251                request
1252                    .headers
1253                    .iter()
1254                    .find(|(n, _)| n.eq_ignore_ascii_case(name))
1255                    .map(|(_, v)| v.clone())
1256            };
1257            let mut stored = self.stored.lock().unwrap_or_else(|e| e.into_inner());
1258            let reply = |status: u16, headers: Vec<(String, String)>| HttpResponse {
1259                status,
1260                headers,
1261                body: Box::new(std::io::empty()),
1262            };
1263            let offset = |n: usize| vec![("upload-offset".to_owned(), n.to_string())];
1264            Ok(match request.method {
1265                HttpMethod::Post => {
1266                    *self.length.lock().unwrap_or_else(|e| e.into_inner()) =
1267                        header("Upload-Length")
1268                            .and_then(|v| v.parse().ok())
1269                            .unwrap_or(0);
1270                    reply(201, vec![("location".to_owned(), "/push/u1".to_owned())])
1271                }
1272                HttpMethod::Head => reply(200, offset(stored.len())),
1273                HttpMethod::Patch => {
1274                    let at: usize = header("Upload-Offset")
1275                        .and_then(|v| v.parse().ok())
1276                        .unwrap_or(usize::MAX);
1277                    if at != stored.len() {
1278                        return Ok(reply(409, vec![]));
1279                    }
1280                    let cut = self
1281                        .patch_cuts
1282                        .lock()
1283                        .unwrap_or_else(|e| e.into_inner())
1284                        .pop_front()
1285                        .flatten();
1286                    match cut {
1287                        Some(keep) => {
1288                            stored.extend_from_slice(&request.body[..keep]);
1289                            return Err(crate::error::Error::from(
1290                                crate::remote::RemoteError::Transport {
1291                                    url: None,
1292                                    detail: "reset".to_owned(),
1293                                },
1294                            ));
1295                        }
1296                        None => {
1297                            stored.extend_from_slice(request.body);
1298                            reply(204, offset(stored.len()))
1299                        }
1300                    }
1301                }
1302                HttpMethod::Get | HttpMethod::Put | HttpMethod::Delete => reply(405, vec![]),
1303            })
1304        }
1305    }
1306
1307    #[test]
1308    fn resumable_error_display_redacts_url_credentials() {
1309        const TOKEN: &str = "grit_bundle_download_secret";
1310        let url = format!("http://user:{TOKEN}@127.0.0.1:8080/.grit/checkouts/demo/bundle");
1311        for text in [
1312            ResumableError::Status {
1313                url: url.clone(),
1314                status: 404,
1315            }
1316            .to_string(),
1317            ResumableError::Protocol {
1318                url: url.clone(),
1319                detail: "no length".to_owned(),
1320            }
1321            .to_string(),
1322            ResumableError::Exhausted {
1323                url: url.clone(),
1324                attempts: 3,
1325                last: "connection reset".to_owned(),
1326            }
1327            .to_string(),
1328        ] {
1329            assert!(!text.contains(TOKEN), "token leaked: {text}");
1330            assert!(!text.contains("user:"), "user leaked: {text}");
1331            assert!(text.contains("127.0.0.1:8080"), "host missing: {text}");
1332        }
1333        assert!(
1334            ResumableError::Exhausted {
1335                url: "http://h/b".to_owned(),
1336                attempts: 1,
1337                last: "x".to_owned(),
1338            }
1339            .to_string()
1340            .contains("http://h/b"),
1341            "URLs without credentials stay unchanged"
1342        );
1343    }
1344
1345    #[test]
1346    fn tus_uploads_resume_from_the_server_offset() {
1347        let dir = tempdir();
1348        let file = dir.path().join("push.bundle");
1349        let data: Vec<u8> = (0..(PATCH_CHUNK * 2 + 1000))
1350            .map(|i| (i % 253) as u8)
1351            .collect();
1352        assert!(std::fs::write(&file, &data).is_ok());
1353        let srv = TusServer {
1354            stored: Mutex::new(Vec::new()),
1355            patch_cuts: Mutex::new([None, Some(1000)].into_iter().collect()),
1356            length: Mutex::new(0),
1357        };
1358        let upload = match TusUpload::create(&srv, "http://h/push", data.len() as u64) {
1359            Ok(u) => u,
1360            Err(e) => panic!("create: {e}"),
1361        };
1362        assert_eq!(upload.url, "http://h/push/u1");
1363        let mut events = Events::default();
1364        assert!(upload.send_file(&srv, &file, &mut events).is_ok());
1365        assert_eq!(*srv.stored.lock().unwrap_or_else(|e| e.into_inner()), data);
1366        assert!(matches!(
1367            events.0.as_slice(),
1368            [
1369                ResumableEvent::Started {
1370                    resumed_from: 0,
1371                    ..
1372                },
1373                ResumableEvent::Reconnecting { attempt: 1, .. },
1374                ResumableEvent::Finished { .. },
1375            ]
1376        ));
1377        assert_eq!(upload.offset(&srv).ok().flatten(), Some(data.len() as u64));
1378    }
1379}