1use std::fs::{File, OpenOptions};
20use std::io::{Read, Seek, SeekFrom, Write};
21use std::path::{Path, PathBuf};
22
23use serde::{Deserialize, Serialize};
24
25use crate::fetch::Progress;
26use crate::progress::{CancelToken, ProgressPhase, ProgressUnit};
27use crate::transport::http::{HttpClient, HttpMethod, HttpRequest, HttpResponse};
28use crate::transport_path::redact_url_for_display;
29
30pub const TUS_VERSION: &str = "1.0.0";
32
33pub const TUS_PATCH_CONTENT_TYPE: &str = "application/offset+octet-stream";
35
36const PATCH_CHUNK: usize = 4 * 1024 * 1024;
38
39const READ_CHUNK: usize = 64 * 1024;
41
42const MAX_ATTEMPTS: u32 = 5;
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47pub enum TransferDirection {
48 Download,
50 Upload,
52}
53
54#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum ResumableEvent {
58 Started {
61 direction: TransferDirection,
63 url: String,
65 total: Option<u64>,
67 resumed_from: u64,
69 },
70 Advanced {
72 done: u64,
74 total: Option<u64>,
76 },
77 Reconnecting {
79 done: u64,
81 attempt: u32,
83 },
84 Finished {
86 done: u64,
88 resumed_from: u64,
90 },
91}
92
93fn resumable_status_message(url: &str, status: u16) -> String {
94 format!("HTTP {status} from {}", redact_url_for_display(url))
95}
96
97fn resumable_protocol_message(url: &str, detail: &str) -> String {
98 format!(
99 "unexpected response from {}: {detail}",
100 redact_url_for_display(url)
101 )
102}
103
104fn resumable_exhausted_message(url: &str, attempts: u32, last: &str) -> String {
105 format!(
106 "the transfer of {} kept failing after {attempts} attempts: {last}",
107 redact_url_for_display(url)
108 )
109}
110
111#[derive(Debug, thiserror::Error)]
113#[non_exhaustive]
114pub enum ResumableError {
115 #[error("{}", resumable_status_message(.url, *status))]
117 Status {
118 url: String,
120 status: u16,
122 },
123 #[error("{}", resumable_protocol_message(.url, .detail))]
125 Protocol {
126 url: String,
128 detail: String,
130 },
131 #[error("{}", resumable_exhausted_message(.url, *attempts, .last))]
133 Exhausted {
134 url: String,
136 attempts: u32,
138 last: String,
140 },
141}
142
143impl From<ResumableError> for crate::error::Error {
144 fn from(err: ResumableError) -> Self {
145 Self::Remote(crate::remote::RemoteError::Resumable(Box::new(err)))
146 }
147}
148
149use crate::error::Result;
150
151#[derive(Debug, Clone, Copy, PartialEq, Eq)]
153pub struct DownloadOutcome {
154 pub bytes: u64,
156 pub resumed_from: u64,
158}
159
160#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
162struct DownloadState {
163 url: String,
165 validator: Option<String>,
168 total: Option<u64>,
170 complete: bool,
172}
173
174fn download_paths(dest: &Path) -> (PathBuf, PathBuf) {
176 let name = dest
177 .file_name()
178 .map(|n| n.to_string_lossy().into_owned())
179 .unwrap_or_default();
180 (
181 dest.with_file_name(format!("{name}.part")),
182 dest.with_file_name(format!("{name}.json")),
183 )
184}
185
186fn load_state(path: &Path) -> Option<DownloadState> {
187 serde_json::from_slice(&std::fs::read(path).ok()?).ok()
188}
189
190fn save_state(path: &Path, state: &DownloadState) -> Result<()> {
191 let tmp = path.with_extension("json.tmp");
192 let bytes = serde_json::to_vec(state).map_err(|e| std::io::Error::other(e.to_string()))?;
193 std::fs::write(&tmp, bytes)?;
194 std::fs::rename(&tmp, path)?;
195 Ok(())
196}
197
198#[derive(Debug)]
200pub enum ResumableDownloadDest {
201 Shared {
203 dest: PathBuf,
205 lock: File,
207 },
208 Private(PathBuf),
210}
211
212pub fn claim_resumable_download(
224 cache_dir: &Path,
225 shared_dest: &Path,
226) -> std::io::Result<ResumableDownloadDest> {
227 std::fs::create_dir_all(cache_dir)?;
228 let lock_path = shared_download_lock_path(shared_dest);
229 let file = OpenOptions::new()
230 .create(true)
231 .read(true)
232 .write(true)
233 .truncate(false)
234 .open(&lock_path)?;
235 match file.try_lock() {
236 Ok(()) => Ok(ResumableDownloadDest::Shared {
237 dest: shared_dest.to_path_buf(),
238 lock: file,
239 }),
240 Err(std::fs::TryLockError::WouldBlock) => Ok(ResumableDownloadDest::Private(
241 private_resumable_download_dest(cache_dir, shared_dest),
242 )),
243 Err(std::fs::TryLockError::Error(e)) => Err(e),
244 }
245}
246
247fn shared_download_lock_path(dest: &Path) -> PathBuf {
248 dest.with_extension("lock")
249}
250
251fn private_resumable_download_dest(cache_dir: &Path, shared_dest: &Path) -> PathBuf {
252 use std::sync::atomic::{AtomicU64, Ordering};
253 static SUFFIX: AtomicU64 = AtomicU64::new(0);
254 let stem = shared_dest
255 .file_stem()
256 .and_then(|s| s.to_str())
257 .unwrap_or("bundle");
258 let n = SUFFIX.fetch_add(1, Ordering::Relaxed);
259 cache_dir.join(format!("{stem}.{}.{n}.private.bundle", std::process::id()))
260}
261
262pub fn discard_download(dest: &Path) -> std::io::Result<()> {
268 let (part, state) = download_paths(dest);
269 for path in [dest, part.as_path(), state.as_path()] {
270 match std::fs::remove_file(path) {
271 Ok(()) => {}
272 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
273 Err(e) => return Err(e),
274 }
275 }
276 Ok(())
277}
278
279pub fn download_resumable(
307 client: &dyn HttpClient,
308 url: &str,
309 dest: &Path,
310 progress: &mut dyn Progress,
311 cancel: &CancelToken,
312) -> Result<DownloadOutcome> {
313 let (part, state_path) = download_paths(dest);
314 let mut state = load_state(&state_path).unwrap_or_default();
315 if state.url != url {
316 discard_download(dest)?;
317 state = DownloadState {
318 url: url.to_owned(),
319 ..DownloadState::default()
320 };
321 }
322 if state.complete {
323 if let (Some(total), Ok(meta)) = (state.total, std::fs::metadata(dest)) {
324 if meta.len() == total {
325 return Ok(DownloadOutcome {
326 bytes: total,
327 resumed_from: total,
328 });
329 }
330 }
331 discard_download(dest)?;
332 state = DownloadState {
333 url: url.to_owned(),
334 ..DownloadState::default()
335 };
336 }
337
338 let resumed_from = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
339 let mut started = false;
340 let mut attempt = 0;
341 let mut last_failure = String::new();
342 loop {
343 if cancel.is_cancelled() {
344 return Err(crate::error::Error::Cancelled);
345 }
346 if attempt > 0 {
347 let done = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
348 if attempt >= MAX_ATTEMPTS {
349 return Err(ResumableError::Exhausted {
350 url: url.to_owned(),
351 attempts: attempt,
352 last: last_failure,
353 }
354 .into());
355 }
356 progress.resumable_transfer(&ResumableEvent::Reconnecting { done, attempt });
357 }
358 attempt += 1;
359 match download_attempt(
360 client,
361 url,
362 &part,
363 &mut state,
364 &state_path,
365 progress,
366 cancel,
367 |p, s| {
368 if !started {
369 started = true;
370 p.start_with_unit(
371 ProgressPhase::TransferringBundle,
372 ProgressUnit::Bytes,
373 s.total,
374 );
375 p.set(s.resumed_from);
376 p.resumable_transfer(&ResumableEvent::Started {
377 direction: TransferDirection::Download,
378 url: url.to_owned(),
379 total: s.total,
380 resumed_from: s.resumed_from,
381 });
382 }
383 },
384 ) {
385 Ok(AttemptOutcome::Complete(bytes)) => {
386 std::fs::rename(&part, dest)?;
387 state.complete = true;
388 state.total = Some(bytes);
389 save_state(&state_path, &state)?;
390 progress.set(bytes);
391 progress.finish();
392 progress.resumable_transfer(&ResumableEvent::Finished {
393 done: bytes,
394 resumed_from: resumed_from.min(bytes),
395 });
396 return Ok(DownloadOutcome {
397 bytes,
398 resumed_from: resumed_from.min(bytes),
399 });
400 }
401 Ok(AttemptOutcome::Dropped(why)) => last_failure = why,
402 Err(e) => return Err(e),
403 }
404 }
405}
406
407struct AttemptStart {
409 total: Option<u64>,
410 resumed_from: u64,
411}
412
413enum AttemptOutcome {
414 Complete(u64),
416 Dropped(String),
418}
419
420#[allow(clippy::too_many_arguments)]
423fn download_attempt(
424 client: &dyn HttpClient,
425 url: &str,
426 part: &Path,
427 state: &mut DownloadState,
428 state_path: &Path,
429 progress: &mut dyn Progress,
430 cancel: &CancelToken,
431 mut on_start: impl FnMut(&mut dyn Progress, &AttemptStart),
432) -> Result<AttemptOutcome> {
433 let have = std::fs::metadata(part).map(|m| m.len()).unwrap_or(0);
434 let mut request = HttpRequest::new(HttpMethod::Get, url).long_running();
435 if have > 0 {
436 request = request.header("Range", format!("bytes={have}-"));
437 if let Some(validator) = &state.validator {
438 request = request.header("If-Range", validator.clone());
439 }
440 }
441 let response = match client.send(&request) {
442 Ok(r) => r,
443 Err(e)
444 if matches!(
445 e,
446 crate::error::Error::Remote(crate::remote::RemoteError::Auth { .. })
447 ) =>
448 {
449 return Err(e);
450 }
451 Err(e) => return Ok(AttemptOutcome::Dropped(e.to_string())),
452 };
453 let (mut offset, total) =
454 match response.status {
455 206 if have > 0 => {
456 let (start, total) = parse_content_range(response.header("content-range"))
457 .ok_or_else(|| ResumableError::Protocol {
458 url: url.to_owned(),
459 detail: "206 without a usable Content-Range".to_owned(),
460 })?;
461 if start != have {
462 return Err(ResumableError::Protocol {
463 url: url.to_owned(),
464 detail: format!("asked for bytes from {have}, got bytes from {start}"),
465 }
466 .into());
467 }
468 (have, total)
469 }
470 200 | 206 => {
471 let total = response
473 .header("content-length")
474 .and_then(|v| v.trim().parse().ok());
475 (0, total)
476 }
477 416 if have > 0 && state.total == Some(have) => {
478 on_start(
479 progress,
480 &AttemptStart {
481 total: state.total,
482 resumed_from: have,
483 },
484 );
485 return Ok(AttemptOutcome::Complete(have));
486 }
487 416 => {
488 File::create(part)?;
490 return Ok(AttemptOutcome::Dropped(
491 "the server's file is shorter than the partial download".to_owned(),
492 ));
493 }
494 status => {
495 return Err(ResumableError::Status {
496 url: url.to_owned(),
497 status,
498 }
499 .into())
500 }
501 };
502 state.total = total;
503 state.validator = validator(&response);
504 save_state(state_path, state)?;
505 on_start(
506 progress,
507 &AttemptStart {
508 total,
509 resumed_from: offset,
510 },
511 );
512
513 let mut file = if offset == 0 {
514 File::create(part)?
515 } else {
516 OpenOptions::new().append(true).open(part)?
517 };
518 let mut body = response.body;
519 let mut buf = vec![0u8; READ_CHUNK];
520 loop {
521 if cancel.is_cancelled() {
522 file.flush()?;
523 return Err(crate::error::Error::Cancelled);
524 }
525 let n = match body.read(&mut buf) {
526 Ok(0) => break,
527 Ok(n) => n,
528 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
529 Err(e) => {
530 file.flush()?;
531 return Ok(AttemptOutcome::Dropped(e.to_string()));
532 }
533 };
534 file.write_all(&buf[..n])?;
535 offset += n as u64;
536 progress.set(offset);
537 progress.resumable_transfer(&ResumableEvent::Advanced {
538 done: offset,
539 total,
540 });
541 }
542 file.flush()?;
543 file.sync_all()?;
544 match total {
545 Some(total) if offset < total => Ok(AttemptOutcome::Dropped(format!(
546 "the connection closed after {offset} of {total} bytes"
547 ))),
548 _ => Ok(AttemptOutcome::Complete(offset)),
549 }
550}
551
552fn validator(response: &HttpResponse) -> Option<String> {
554 response
555 .header("etag")
556 .filter(|v| !v.starts_with("W/"))
557 .or_else(|| response.header("last-modified"))
558 .map(str::to_owned)
559}
560
561fn parse_content_range(value: Option<&str>) -> Option<(u64, Option<u64>)> {
563 let rest = value?.trim().strip_prefix("bytes ")?;
564 let (range, total) = rest.split_once('/')?;
565 let (start, _end) = range.split_once('-')?;
566 let start = start.trim().parse().ok()?;
567 let total = match total.trim() {
568 "*" => None,
569 t => Some(t.parse().ok()?),
570 };
571 Some((start, total))
572}
573
574#[derive(Debug, Clone, PartialEq, Eq)]
581pub(crate) struct TusUpload {
582 pub url: String,
584 pub length: u64,
586}
587
588impl TusUpload {
589 pub fn create(client: &dyn HttpClient, endpoint: &str, length: u64) -> Result<Self> {
597 let request = HttpRequest::new(HttpMethod::Post, endpoint)
598 .header("Tus-Resumable", TUS_VERSION)
599 .header("Upload-Length", length.to_string());
600 let response = client.send(&request)?;
601 if response.status != 201 {
602 return Err(ResumableError::Status {
603 url: endpoint.to_owned(),
604 status: response.status,
605 }
606 .into());
607 }
608 let location = response
609 .header("location")
610 .ok_or_else(|| ResumableError::Protocol {
611 url: endpoint.to_owned(),
612 detail: "no Location for the new upload".to_owned(),
613 })?;
614 let url = resolve_url(endpoint, location).ok_or_else(|| ResumableError::Protocol {
615 url: endpoint.to_owned(),
616 detail: format!("bad Location {location:?}"),
617 })?;
618 Ok(Self { url, length })
619 }
620
621 pub fn offset(&self, client: &dyn HttpClient) -> Result<Option<u64>> {
629 let request =
630 HttpRequest::new(HttpMethod::Head, &self.url).header("Tus-Resumable", TUS_VERSION);
631 let response = client.send(&request)?;
632 match response.status {
633 404 | 410 => Ok(None),
634 s if (200..300).contains(&s) => upload_offset(&response, &self.url).map(Some),
635 status => Err(ResumableError::Status {
636 url: self.url.clone(),
637 status,
638 }
639 .into()),
640 }
641 }
642
643 pub fn send_file(
652 &self,
653 client: &dyn HttpClient,
654 file: &Path,
655 progress: &mut dyn Progress,
656 ) -> Result<()> {
657 let gone = |status| ResumableError::Status {
658 url: self.url.clone(),
659 status,
660 };
661 let mut offset = self.offset(client)?.ok_or_else(|| gone(404))?;
662 let resumed_from = offset;
663 progress.start_with_unit(
664 ProgressPhase::TransferringBundle,
665 ProgressUnit::Bytes,
666 Some(self.length),
667 );
668 progress.set(offset);
669 progress.resumable_transfer(&ResumableEvent::Started {
670 direction: TransferDirection::Upload,
671 url: self.url.clone(),
672 total: Some(self.length),
673 resumed_from,
674 });
675 let mut input = File::open(file)?;
676 let mut buf = vec![0u8; PATCH_CHUNK];
677 let mut failures = 0;
678 while offset < self.length {
679 let n = usize::try_from(self.length - offset)
680 .map_or(PATCH_CHUNK, |left| left.min(PATCH_CHUNK));
681 input.seek(SeekFrom::Start(offset))?;
682 input.read_exact(&mut buf[..n])?;
683 let request = HttpRequest::new(HttpMethod::Patch, &self.url)
684 .header("Tus-Resumable", TUS_VERSION)
685 .header("Upload-Offset", offset.to_string())
686 .header("Content-Type", TUS_PATCH_CONTENT_TYPE)
687 .body(&buf[..n])
688 .long_running();
689 let failure = match client.send(&request) {
690 Ok(r) if r.status == 204 || r.status == 200 => {
691 offset = upload_offset(&r, &self.url)?;
692 failures = 0;
693 progress.set(offset);
694 progress.resumable_transfer(&ResumableEvent::Advanced {
695 done: offset,
696 total: Some(self.length),
697 });
698 continue;
699 }
700 Ok(r) if r.status == 404 || r.status == 410 => {
701 return Err(crate::error::Error::from(gone(r.status)))
702 }
703 Ok(r) if r.status == 409 => format!("offset {offset} was out of date"),
704 Ok(r) if (400..500).contains(&r.status) => {
705 return Err(crate::error::Error::from(gone(r.status)))
706 }
707 Ok(r) => format!("HTTP {}", r.status),
708 Err(e)
709 if matches!(
710 e,
711 crate::error::Error::Remote(crate::remote::RemoteError::Auth { .. })
712 ) =>
713 {
714 return Err(e);
715 }
716 Err(e) => e.to_string(),
717 };
718 failures += 1;
719 if failures >= MAX_ATTEMPTS {
720 return Err(ResumableError::Exhausted {
721 url: self.url.clone(),
722 attempts: failures,
723 last: failure,
724 }
725 .into());
726 }
727 progress.resumable_transfer(&ResumableEvent::Reconnecting {
728 done: offset,
729 attempt: failures,
730 });
731 if let Ok(Some(server)) = self.offset(client) {
733 offset = server;
734 }
735 }
736 progress.set(self.length);
737 progress.finish();
738 progress.resumable_transfer(&ResumableEvent::Finished {
739 done: self.length,
740 resumed_from,
741 });
742 Ok(())
743 }
744}
745
746fn upload_offset(response: &HttpResponse, url: &str) -> Result<u64> {
747 Ok(response
748 .header("upload-offset")
749 .and_then(|v| v.trim().parse().ok())
750 .ok_or_else(|| ResumableError::Protocol {
751 url: url.to_owned(),
752 detail: "no Upload-Offset".to_owned(),
753 })?)
754}
755
756#[must_use]
758pub(crate) fn resolve_url(base: &str, reference: &str) -> Option<String> {
759 if let Ok(absolute) = url::Url::parse(reference) {
760 return Some(absolute.to_string());
761 }
762 url::Url::parse(base)
763 .ok()?
764 .join(reference)
765 .ok()
766 .map(|u| u.to_string())
767}
768
769#[cfg(test)]
770mod tests {
771 use super::*;
772 use crate::fetch::NoProgress;
773 use crate::progress::CancelToken;
774 use crate::remote::RemoteError;
775 use std::collections::VecDeque;
776 use std::sync::Mutex;
777
778 struct FileServer {
780 data: Vec<u8>,
781 etag: String,
782 cuts: Mutex<VecDeque<usize>>,
784 ranges: Mutex<Vec<Option<String>>>,
785 }
786
787 struct Cut {
788 data: std::io::Cursor<Vec<u8>>,
789 fail_after: bool,
790 }
791
792 impl Read for Cut {
793 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
794 let n = self.data.read(buf)?;
795 if n == 0 && self.fail_after {
796 return Err(std::io::Error::new(
797 std::io::ErrorKind::ConnectionReset,
798 "reset",
799 ));
800 }
801 Ok(n)
802 }
803 }
804
805 impl HttpClient for FileServer {
806 fn get(&self, _: &str, _: Option<&str>) -> crate::error::Result<Vec<u8>> {
807 unreachable!()
808 }
809 fn post(
810 &self,
811 _: &str,
812 _: &str,
813 _: &str,
814 _: &[u8],
815 _: Option<&str>,
816 ) -> crate::error::Result<Vec<u8>> {
817 unreachable!()
818 }
819 fn send(&self, request: &HttpRequest<'_>) -> crate::error::Result<HttpResponse> {
820 let range = request
821 .headers
822 .iter()
823 .find(|(n, _)| *n == "Range")
824 .map(|(_, v)| v.clone());
825 let if_range = request
826 .headers
827 .iter()
828 .find(|(n, _)| *n == "If-Range")
829 .map(|(_, v)| v.clone());
830 self.ranges
831 .lock()
832 .unwrap_or_else(|e| e.into_inner())
833 .push(range.clone());
834 let start = range
835 .as_deref()
836 .and_then(|r| r.strip_prefix("bytes="))
837 .and_then(|r| r.strip_suffix('-'))
838 .and_then(|r| r.parse::<usize>().ok())
839 .filter(|_| if_range.as_deref().is_none_or(|v| v == self.etag));
840 let total = self.data.len();
841 let (status, mut headers, body) = match start {
842 Some(s) if s >= total => (416, vec![], vec![]),
843 Some(s) => (
844 206,
845 vec![(
846 "content-range".to_owned(),
847 format!("bytes {s}-{}/{total}", total - 1),
848 )],
849 self.data[s..].to_vec(),
850 ),
851 None => (
852 200,
853 vec![("content-length".to_owned(), total.to_string())],
854 self.data.clone(),
855 ),
856 };
857 headers.push(("etag".to_owned(), self.etag.clone()));
858 let cut = self
859 .cuts
860 .lock()
861 .unwrap_or_else(|e| e.into_inner())
862 .pop_front();
863 let (body, fail_after) = match cut {
864 Some(n) if n < body.len() => (body[..n].to_vec(), true),
865 _ => (body, false),
866 };
867 Ok(HttpResponse {
868 status,
869 headers,
870 body: Box::new(Cut {
871 data: std::io::Cursor::new(body),
872 fail_after,
873 }),
874 })
875 }
876 }
877
878 #[derive(Default)]
879 struct Events(Vec<ResumableEvent>);
880 impl crate::progress::ProgressSink for Events {}
881 impl Progress for Events {
882 fn resumable_transfer(&mut self, event: &ResumableEvent) {
883 if !matches!(event, ResumableEvent::Advanced { .. }) {
884 self.0.push(event.clone());
885 }
886 }
887 }
888
889 fn server(cuts: &[usize]) -> FileServer {
890 FileServer {
891 data: (0..200_000u32).map(|i| (i % 251) as u8).collect(),
892 etag: "\"v1\"".to_owned(),
893 cuts: Mutex::new(cuts.iter().copied().collect()),
894 ranges: Mutex::new(Vec::new()),
895 }
896 }
897
898 fn tempdir() -> tempfile::TempDir {
899 match tempfile::tempdir() {
900 Ok(d) => d,
901 Err(e) => panic!("tempdir: {e}"),
902 }
903 }
904
905 #[test]
906 fn dropped_downloads_resume_in_the_same_run() {
907 let dir = tempdir();
908 let dest = dir.path().join("clone.bundle");
909 let srv = server(&[70_000, 10_000]);
910 let mut events = Events::default();
911 let outcome = download_resumable(
912 &srv,
913 "http://h/b",
914 &dest,
915 &mut events,
916 &CancelToken::never(),
917 );
918 assert_eq!(
919 outcome.ok(),
920 Some(DownloadOutcome {
921 bytes: 200_000,
922 resumed_from: 0
923 })
924 );
925 assert_eq!(std::fs::read(&dest).ok(), Some(srv.data.clone()));
926 let ranges = srv.ranges.lock().unwrap_or_else(|e| e.into_inner()).clone();
927 assert_eq!(
928 ranges,
929 [
930 None,
931 Some("bytes=70000-".to_owned()),
932 Some("bytes=80000-".to_owned())
933 ]
934 );
935 assert!(matches!(
936 events.0.as_slice(),
937 [
938 ResumableEvent::Started {
939 resumed_from: 0,
940 total: Some(200_000),
941 ..
942 },
943 ResumableEvent::Reconnecting {
944 done: 70_000,
945 attempt: 1
946 },
947 ResumableEvent::Reconnecting {
948 done: 80_000,
949 attempt: 2
950 },
951 ResumableEvent::Finished {
952 done: 200_000,
953 resumed_from: 0
954 },
955 ]
956 ));
957 }
958
959 #[test]
960 fn a_later_run_resumes_a_partial_download() {
961 let dir = tempdir();
962 let dest = dir.path().join("clone.bundle");
963 let failing = server(&[50_000, 0, 0, 0, 0, 0]);
965 assert!(matches!(
966 download_resumable(
967 &failing,
968 "http://h/b",
969 &dest,
970 &mut NoProgress,
971 &CancelToken::never()
972 ),
973 Err(crate::error::Error::Remote(crate::remote::RemoteError::Resumable(ref r)))
974 if matches!(**r, ResumableError::Exhausted { .. })
975 ));
976 assert!(!dest.exists());
977
978 let srv = server(&[]);
979 let mut events = Events::default();
980 let outcome = download_resumable(
981 &srv,
982 "http://h/b",
983 &dest,
984 &mut events,
985 &CancelToken::never(),
986 );
987 assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(50_000));
988 assert_eq!(std::fs::read(&dest).ok(), Some(srv.data.clone()));
989 assert!(matches!(
990 events.0.first(),
991 Some(ResumableEvent::Started {
992 resumed_from: 50_000,
993 ..
994 })
995 ));
996
997 let again = server(&[]);
999 let outcome = download_resumable(
1000 &again,
1001 "http://h/b",
1002 &dest,
1003 &mut NoProgress,
1004 &CancelToken::never(),
1005 );
1006 assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(200_000));
1007 assert!(again
1008 .ranges
1009 .lock()
1010 .unwrap_or_else(|e| e.into_inner())
1011 .is_empty());
1012 }
1013
1014 #[test]
1015 fn a_changed_file_or_url_starts_over() {
1016 let dir = tempdir();
1017 let dest = dir.path().join("clone.bundle");
1018 let failing = server(&[50_000, 0, 0, 0, 0, 0]);
1019 let _ = download_resumable(
1020 &failing,
1021 "http://h/b",
1022 &dest,
1023 &mut NoProgress,
1024 &CancelToken::never(),
1025 );
1026
1027 let mut changed = server(&[]);
1028 changed.etag = "\"v2\"".to_owned();
1029 let outcome = download_resumable(
1030 &changed,
1031 "http://h/b",
1032 &dest,
1033 &mut NoProgress,
1034 &CancelToken::never(),
1035 );
1036 assert_eq!(outcome.ok().map(|o| o.bytes), Some(200_000));
1037 assert_eq!(std::fs::read(&dest).ok(), Some(changed.data.clone()));
1038
1039 let other = server(&[]);
1040 let outcome = download_resumable(
1041 &other,
1042 "http://h/other",
1043 &dest,
1044 &mut NoProgress,
1045 &CancelToken::never(),
1046 );
1047 assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(0));
1048 assert_eq!(
1049 other
1050 .ranges
1051 .lock()
1052 .unwrap_or_else(|e| e.into_inner())
1053 .as_slice(),
1054 [None]
1055 );
1056 }
1057
1058 struct CancelAfter {
1060 flag: CancelToken,
1061 after: u64,
1062 }
1063 impl crate::progress::ProgressSink for CancelAfter {}
1064 impl Progress for CancelAfter {
1065 fn resumable_transfer(&mut self, event: &ResumableEvent) {
1066 if let ResumableEvent::Advanced { done, .. } = event {
1067 if *done >= self.after {
1068 self.flag.cancel();
1069 }
1070 }
1071 }
1072 }
1073
1074 #[test]
1075 fn a_cancelled_download_stops_mid_body_and_resumes_next_time() {
1076 let dir = tempdir();
1077 let dest = dir.path().join("clone.bundle");
1078 let srv = server(&[]);
1079 let flag = CancelToken::new();
1080 let mut progress = CancelAfter {
1081 flag: flag.clone(),
1082 after: 1,
1083 };
1084 assert!(matches!(
1085 download_resumable(&srv, "http://h/b", &dest, &mut progress, &flag),
1086 Err(crate::error::Error::Cancelled)
1087 ));
1088 assert!(!dest.exists());
1089 let (part, _) = download_paths(&dest);
1090 let kept = std::fs::metadata(&part).map(|m| m.len()).unwrap_or(0);
1091 assert!(
1092 kept > 0 && kept < 200_000,
1093 "one chunk is kept, got {kept} bytes"
1094 );
1095
1096 let outcome = download_resumable(
1097 &srv,
1098 "http://h/b",
1099 &dest,
1100 &mut NoProgress,
1101 &CancelToken::never(),
1102 );
1103 assert_eq!(outcome.ok().map(|o| o.resumed_from), Some(kept));
1104 assert_eq!(std::fs::read(&dest).ok(), Some(srv.data.clone()));
1105 }
1106
1107 #[test]
1108 fn claim_resumable_download_serializes_the_shared_slot() {
1109 let dir = tempdir();
1110 let shared = dir.path().join("abc.bundle");
1111 let first = claim_resumable_download(dir.path(), &shared).expect("first claim");
1112 let (first_dest, first_lock) = match first {
1113 ResumableDownloadDest::Shared { dest, lock } => (dest, lock),
1114 ResumableDownloadDest::Private(_) => panic!("first claim should be shared"),
1115 };
1116 assert_eq!(first_dest, shared);
1117 let second = claim_resumable_download(dir.path(), &shared).expect("second claim");
1118 assert!(
1119 matches!(second, ResumableDownloadDest::Private(_)),
1120 "second claim must not touch the shared partial files"
1121 );
1122 let other = std::fs::File::open(shared_download_lock_path(&shared)).expect("lock file");
1123 assert!(
1124 matches!(other.try_lock(), Err(std::fs::TryLockError::WouldBlock)),
1125 "lock file stays held until the first guard drops"
1126 );
1127 drop(first_lock);
1128 }
1129
1130 #[test]
1131 fn auth_failure_does_not_retry_the_download() {
1132 use crate::error::AuthFailure;
1133
1134 struct AuthClient;
1135 impl HttpClient for AuthClient {
1136 fn get(&self, _: &str, _: Option<&str>) -> crate::error::Result<Vec<u8>> {
1137 unreachable!()
1138 }
1139 fn post(
1140 &self,
1141 _: &str,
1142 _: &str,
1143 _: &str,
1144 _: &[u8],
1145 _: Option<&str>,
1146 ) -> crate::error::Result<Vec<u8>> {
1147 unreachable!()
1148 }
1149 fn send(&self, _: &HttpRequest<'_>) -> crate::error::Result<HttpResponse> {
1150 Err(RemoteError::Auth {
1151 url: "http://h/b".to_owned(),
1152 reason: AuthFailure::Rejected,
1153 }
1154 .into())
1155 }
1156 }
1157
1158 let dir = tempdir();
1159 let dest = dir.path().join("clone.bundle");
1160 let outcome = download_resumable(
1161 &AuthClient,
1162 "http://h/b",
1163 &dest,
1164 &mut NoProgress,
1165 &CancelToken::never(),
1166 );
1167 assert!(
1168 matches!(
1169 outcome,
1170 Err(crate::error::Error::Remote(
1171 crate::remote::RemoteError::Auth { .. }
1172 ))
1173 ),
1174 "auth errors must fail immediately: {outcome:?}"
1175 );
1176 assert!(
1177 !matches!(
1178 &outcome,
1179 Err(crate::error::Error::Remote(crate::remote::RemoteError::Resumable(r)))
1180 if matches!(**r, ResumableError::Exhausted { .. })
1181 ),
1182 "auth must not be retried like a dropped connection"
1183 );
1184 }
1185
1186 #[test]
1187 fn an_already_cancelled_download_sends_no_request() {
1188 let dir = tempdir();
1189 let dest = dir.path().join("clone.bundle");
1190 let srv = server(&[]);
1191 let flag = CancelToken::new();
1192 flag.cancel();
1193 assert!(matches!(
1194 download_resumable(&srv, "http://h/b", &dest, &mut NoProgress, &flag),
1195 Err(crate::error::Error::Cancelled)
1196 ));
1197 assert!(srv
1198 .ranges
1199 .lock()
1200 .unwrap_or_else(|e| e.into_inner())
1201 .is_empty());
1202 }
1203
1204 #[test]
1205 fn content_ranges_parse() {
1206 assert_eq!(
1207 parse_content_range(Some("bytes 10-99/100")),
1208 Some((10, Some(100)))
1209 );
1210 assert_eq!(parse_content_range(Some("bytes 10-99/*")), Some((10, None)));
1211 assert_eq!(parse_content_range(Some("items 1-2/3")), None);
1212 assert_eq!(parse_content_range(None), None);
1213 }
1214
1215 #[test]
1216 fn urls_resolve_against_the_endpoint() {
1217 assert_eq!(
1218 resolve_url("http://h:1/repo/push", "/repo/push/abc").as_deref(),
1219 Some("http://h:1/repo/push/abc")
1220 );
1221 assert_eq!(
1222 resolve_url("http://h/repo/push", "https://other/x").as_deref(),
1223 Some("https://other/x")
1224 );
1225 }
1226
1227 struct TusServer {
1229 stored: Mutex<Vec<u8>>,
1230 patch_cuts: Mutex<VecDeque<Option<usize>>>,
1232 length: Mutex<u64>,
1233 }
1234
1235 impl HttpClient for TusServer {
1236 fn get(&self, _: &str, _: Option<&str>) -> crate::error::Result<Vec<u8>> {
1237 unreachable!()
1238 }
1239 fn post(
1240 &self,
1241 _: &str,
1242 _: &str,
1243 _: &str,
1244 _: &[u8],
1245 _: Option<&str>,
1246 ) -> crate::error::Result<Vec<u8>> {
1247 unreachable!()
1248 }
1249 fn send(&self, request: &HttpRequest<'_>) -> crate::error::Result<HttpResponse> {
1250 let header = |name: &str| {
1251 request
1252 .headers
1253 .iter()
1254 .find(|(n, _)| n.eq_ignore_ascii_case(name))
1255 .map(|(_, v)| v.clone())
1256 };
1257 let mut stored = self.stored.lock().unwrap_or_else(|e| e.into_inner());
1258 let reply = |status: u16, headers: Vec<(String, String)>| HttpResponse {
1259 status,
1260 headers,
1261 body: Box::new(std::io::empty()),
1262 };
1263 let offset = |n: usize| vec![("upload-offset".to_owned(), n.to_string())];
1264 Ok(match request.method {
1265 HttpMethod::Post => {
1266 *self.length.lock().unwrap_or_else(|e| e.into_inner()) =
1267 header("Upload-Length")
1268 .and_then(|v| v.parse().ok())
1269 .unwrap_or(0);
1270 reply(201, vec![("location".to_owned(), "/push/u1".to_owned())])
1271 }
1272 HttpMethod::Head => reply(200, offset(stored.len())),
1273 HttpMethod::Patch => {
1274 let at: usize = header("Upload-Offset")
1275 .and_then(|v| v.parse().ok())
1276 .unwrap_or(usize::MAX);
1277 if at != stored.len() {
1278 return Ok(reply(409, vec![]));
1279 }
1280 let cut = self
1281 .patch_cuts
1282 .lock()
1283 .unwrap_or_else(|e| e.into_inner())
1284 .pop_front()
1285 .flatten();
1286 match cut {
1287 Some(keep) => {
1288 stored.extend_from_slice(&request.body[..keep]);
1289 return Err(crate::error::Error::from(
1290 crate::remote::RemoteError::Transport {
1291 url: None,
1292 detail: "reset".to_owned(),
1293 },
1294 ));
1295 }
1296 None => {
1297 stored.extend_from_slice(request.body);
1298 reply(204, offset(stored.len()))
1299 }
1300 }
1301 }
1302 HttpMethod::Get | HttpMethod::Put | HttpMethod::Delete => reply(405, vec![]),
1303 })
1304 }
1305 }
1306
1307 #[test]
1308 fn resumable_error_display_redacts_url_credentials() {
1309 const TOKEN: &str = "grit_bundle_download_secret";
1310 let url = format!("http://user:{TOKEN}@127.0.0.1:8080/.grit/checkouts/demo/bundle");
1311 for text in [
1312 ResumableError::Status {
1313 url: url.clone(),
1314 status: 404,
1315 }
1316 .to_string(),
1317 ResumableError::Protocol {
1318 url: url.clone(),
1319 detail: "no length".to_owned(),
1320 }
1321 .to_string(),
1322 ResumableError::Exhausted {
1323 url: url.clone(),
1324 attempts: 3,
1325 last: "connection reset".to_owned(),
1326 }
1327 .to_string(),
1328 ] {
1329 assert!(!text.contains(TOKEN), "token leaked: {text}");
1330 assert!(!text.contains("user:"), "user leaked: {text}");
1331 assert!(text.contains("127.0.0.1:8080"), "host missing: {text}");
1332 }
1333 assert!(
1334 ResumableError::Exhausted {
1335 url: "http://h/b".to_owned(),
1336 attempts: 1,
1337 last: "x".to_owned(),
1338 }
1339 .to_string()
1340 .contains("http://h/b"),
1341 "URLs without credentials stay unchanged"
1342 );
1343 }
1344
1345 #[test]
1346 fn tus_uploads_resume_from_the_server_offset() {
1347 let dir = tempdir();
1348 let file = dir.path().join("push.bundle");
1349 let data: Vec<u8> = (0..(PATCH_CHUNK * 2 + 1000))
1350 .map(|i| (i % 253) as u8)
1351 .collect();
1352 assert!(std::fs::write(&file, &data).is_ok());
1353 let srv = TusServer {
1354 stored: Mutex::new(Vec::new()),
1355 patch_cuts: Mutex::new([None, Some(1000)].into_iter().collect()),
1356 length: Mutex::new(0),
1357 };
1358 let upload = match TusUpload::create(&srv, "http://h/push", data.len() as u64) {
1359 Ok(u) => u,
1360 Err(e) => panic!("create: {e}"),
1361 };
1362 assert_eq!(upload.url, "http://h/push/u1");
1363 let mut events = Events::default();
1364 assert!(upload.send_file(&srv, &file, &mut events).is_ok());
1365 assert_eq!(*srv.stored.lock().unwrap_or_else(|e| e.into_inner()), data);
1366 assert!(matches!(
1367 events.0.as_slice(),
1368 [
1369 ResumableEvent::Started {
1370 resumed_from: 0,
1371 ..
1372 },
1373 ResumableEvent::Reconnecting { attempt: 1, .. },
1374 ResumableEvent::Finished { .. },
1375 ]
1376 ));
1377 assert_eq!(upload.offset(&srv).ok().flatten(), Some(data.len() as u64));
1378 }
1379}