1use super::{TransferEvent, TransferJob, TransferStatus, TransferType};
2use crate::credentials::{Profile, ProfileManager};
3use crate::s3::{
4 plan_multipart_upload, MultipartUploadGuard, S3ClientManager, MULTIPART_UPLOAD_THRESHOLD,
5};
6use aws_sdk_s3::error::DisplayErrorContext;
7use aws_sdk_s3::primitives::ByteStream;
8use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
9use aws_sdk_s3::Client;
10use aws_smithy_types::byte_stream::Length;
11use std::collections::{HashMap, VecDeque};
12use std::sync::atomic::{AtomicUsize, Ordering};
13use std::sync::Arc;
14use std::time::Duration;
15use tauri::{AppHandle, Emitter};
16use tokio::sync::{Mutex, Notify, RwLock};
17
18const MIN_TRANSFER_TIMEOUT: Duration = Duration::from_secs(60);
19const MAX_TRANSFER_TIMEOUT: Duration = Duration::from_secs(30 * 60);
20const MIN_ASSUMED_THROUGHPUT_BYTES_PER_SEC: u64 = 100 * 1024;
21const METADATA_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
22
23fn transfer_timeout_for(size_bytes: u64) -> Duration {
24 let scaled_secs = size_bytes.div_ceil(MIN_ASSUMED_THROUGHPUT_BYTES_PER_SEC);
25 Duration::from_secs(scaled_secs).clamp(MIN_TRANSFER_TIMEOUT, MAX_TRANSFER_TIMEOUT)
26}
27
28pub struct TransferManager {
30 jobs: Arc<RwLock<HashMap<String, TransferJob>>>,
31 queue: Arc<Mutex<VecDeque<String>>>, abort_handles: Arc<RwLock<HashMap<String, tokio::task::AbortHandle>>>,
33 max_concurrency: Arc<AtomicUsize>,
34 active_count: Arc<AtomicUsize>,
35 slot_notify: Arc<Notify>,
36 app_handle: Arc<RwLock<Option<AppHandle>>>,
37 journal: Mutex<Option<std::path::PathBuf>>,
38}
39
40struct ActiveSlotGuard {
41 active_count: Arc<AtomicUsize>,
42 slot_notify: Arc<Notify>,
43}
44
45impl Drop for ActiveSlotGuard {
46 fn drop(&mut self) {
47 self.active_count.fetch_sub(1, Ordering::AcqRel);
48 self.slot_notify.notify_waiters();
49 }
50}
51
52impl Default for TransferManager {
53 fn default() -> Self {
54 Self::new()
55 }
56}
57
58impl TransferManager {
59 pub fn new() -> Self {
60 Self {
61 jobs: Arc::new(RwLock::new(HashMap::new())),
62 queue: Arc::new(Mutex::new(VecDeque::new())),
63 abort_handles: Arc::new(RwLock::new(HashMap::new())),
64 max_concurrency: Arc::new(AtomicUsize::new(5)),
65 active_count: Arc::new(AtomicUsize::new(0)),
66 slot_notify: Arc::new(Notify::new()),
67 app_handle: Arc::new(RwLock::new(None)),
68 journal: Mutex::new(None),
69 }
70 }
71
72 pub async fn set_app_handle(&self, app_handle: AppHandle) {
73 let mut handle = self.app_handle.write().await;
74 *handle = Some(app_handle);
75 }
76
77 pub async fn configure_journal(&self, path: std::path::PathBuf) -> crate::error::Result<()> {
78 *self.journal.lock().await = Some(path.clone());
79 if path.exists() {
80 let content = std::fs::read(&path)?;
81 let mut restored: Vec<TransferJob> = match serde_json::from_slice(&content) {
82 Ok(jobs) => jobs,
83 Err(error) => {
84 let backup = path
85 .with_file_name(format!("transfers.invalid-{}.json", uuid::Uuid::new_v4()));
86 std::fs::rename(&path, &backup)?;
87 log::error!(
88 "Invalid transfer journal preserved at {}: {error}",
89 backup.display()
90 );
91 Vec::new()
92 }
93 };
94 for job in &mut restored {
95 job.bytes_per_second = 0.0;
96 if matches!(
97 job.status,
98 TransferStatus::Pending | TransferStatus::InProgress
99 ) {
100 job.status = TransferStatus::Failed(match job.transfer_type {
101 TransferType::Upload => "Interrupted when the app closed. Check the destination before retrying; existing objects will not be replaced.".into(),
102 TransferType::Download => "Interrupted when the app closed. Queue the download again to choose its destination.".into(),
103 });
104 job.finished_at = Some(chrono::Utc::now().timestamp_millis());
105 }
106 }
107 *self.jobs.write().await = restored
108 .into_iter()
109 .map(|job| (job.id.clone(), job))
110 .collect();
111 }
112 self.persist_jobs().await
113 }
114
115 async fn persist_jobs(&self) -> crate::error::Result<()> {
116 let journal = self.journal.lock().await;
118 let Some(path) = journal.as_ref() else {
119 return Ok(());
120 };
121 let mut jobs = self.jobs.write().await;
122 let mut finished: Vec<_> = jobs
123 .values()
124 .filter(|job| {
125 matches!(
126 job.status,
127 TransferStatus::Completed
128 | TransferStatus::Failed(_)
129 | TransferStatus::Cancelled
130 )
131 })
132 .map(|job| (job.finished_at.unwrap_or(job.created_at), job.id.clone()))
133 .collect();
134 finished.sort_unstable();
135 let excess = finished.len().saturating_sub(1000);
136 for (_, id) in finished.into_iter().take(excess) {
137 jobs.remove(&id);
138 }
139 let bytes = serde_json::to_vec(&jobs.values().collect::<Vec<_>>())?;
140 drop(jobs);
141 let path = path.clone();
142 tokio::task::spawn_blocking(move || crate::credentials::write_private_file(&path, &bytes))
143 .await
144 .map_err(|error| crate::error::AppError::IoError(error.to_string()))??;
145 Ok(())
146 }
147
148 async fn persist_or_log(&self) {
149 if let Err(error) = self.persist_jobs().await {
150 log::error!("Transfer recovery journal could not be saved: {error}");
151 }
152 }
153
154 pub async fn add_job(&self, job: TransferJob) {
155 self.add_jobs(vec![job]).await;
156 }
157
158 pub async fn add_jobs(&self, added: Vec<TransferJob>) {
159 {
160 let mut jobs = self.jobs.write().await;
161 for job in &added {
162 jobs.insert(job.id.clone(), job.clone());
163 }
164 }
165 if let Err(error) = self.persist_jobs().await {
166 let mut jobs = self.jobs.write().await;
167 for job in &added {
168 if let Some(current) = jobs.get_mut(&job.id) {
169 current.status =
170 TransferStatus::Failed(format!("Could not save queued transfer: {error}"));
171 }
172 }
173 return;
174 }
175 {
176 let mut queue = self.queue.lock().await;
177 queue.extend(added.iter().map(|job| job.id.clone()));
178 }
179 for job in added {
180 if let Some(app) = self.app_handle.read().await.as_ref() {
181 let _ = app.emit("transfer-added", &job);
182 }
183 self.emit_update(&job).await;
184 }
185 }
186
187 pub fn set_max_concurrency(&self, max: usize) {
188 let clamped = max.clamp(1, 20);
189 self.max_concurrency.store(clamped, Ordering::Release);
190 self.slot_notify.notify_waiters();
191 }
192
193 pub async fn get_job(&self, id: &str) -> Option<TransferJob> {
194 let jobs = self.jobs.read().await;
195 jobs.get(id).cloned()
196 }
197
198 pub async fn list_jobs(&self) -> Vec<TransferJob> {
199 let jobs = self.jobs.read().await;
200 let mut list: Vec<TransferJob> = jobs.values().cloned().collect();
201 list.sort_by_key(|job| std::cmp::Reverse(job.created_at)); list
203 }
204
205 pub async fn cancel_job(&self, id: &str) -> bool {
207 let mut jobs = self.jobs.write().await;
208 if let Some(job) = jobs.get_mut(id) {
209 match job.status {
211 TransferStatus::Pending | TransferStatus::InProgress => {
212 job.status = TransferStatus::Cancelled;
213 job.bytes_per_second = 0.0;
214 let job_clone = job.clone();
215 let needs_graceful_cleanup = Self::is_multipart_upload_job(job);
216
217 {
218 let mut queue = self.queue.lock().await;
219 queue.retain(|job_id| job_id != id);
220 }
221
222 let mut handles = self.abort_handles.write().await;
225 if let Some(handle) = handles.remove(id) {
226 if needs_graceful_cleanup {
227 log::info!(
228 "Cancellation requested for multipart job {}; waiting for the current part so the server-side upload can be aborted cleanly",
229 id
230 );
231 } else {
232 handle.abort();
233 log::info!("Aborted job task: {}", id);
234 }
235 }
236
237 drop(handles);
238 drop(jobs);
239 self.persist_or_log().await;
240 self.emit_update(&job_clone).await;
241 return true;
242 }
243 _ => return false,
244 }
245 }
246 false
247 }
248
249 pub async fn remove_job(&self, id: &str) -> bool {
251 let needs_graceful_cleanup = self
252 .get_job(id)
253 .await
254 .as_ref()
255 .is_some_and(Self::is_multipart_upload_job);
256
257 {
258 let mut queue = self.queue.lock().await;
259 queue.retain(|job_id| job_id != id);
260 }
261
262 let mut handles = self.abort_handles.write().await;
263 if let Some(handle) = handles.remove(id) {
264 if !needs_graceful_cleanup {
265 handle.abort();
266 }
267 }
268 drop(handles);
269
270 let mut jobs = self.jobs.write().await;
271 let removed = jobs.remove(id).is_some();
272 drop(jobs);
273 self.persist_or_log().await;
274 removed
275 }
276
277 pub async fn clear_completed(&self) -> usize {
279 let mut jobs = self.jobs.write().await;
280 let initial_count = jobs.len();
281 jobs.retain(|_, job| {
282 matches!(
283 job.status,
284 TransferStatus::Pending | TransferStatus::InProgress
285 )
286 });
287 let removed = initial_count - jobs.len();
288 drop(jobs);
289 self.persist_or_log().await;
290 removed
291 }
292
293 pub async fn retry_job(&self, id: &str) -> Option<String> {
295 let jobs = self.jobs.read().await;
296 if let Some(job) = jobs.get(id) {
297 match &job.status {
299 TransferStatus::Failed(_) | TransferStatus::Cancelled => {
300 let mut new_job = TransferJob::new(
302 job.transfer_type.clone(),
303 job.profile_id.clone(),
304 job.bucket.clone(),
305 job.bucket_region.clone(),
306 job.key.clone(),
307 std::path::PathBuf::from(&job.local_path),
308 job.total_bytes,
309 );
310
311 new_job.download_destination = job.download_destination.clone();
312
313 new_job.parent_group_id = job.parent_group_id.clone();
315 new_job.group_name = job.group_name.clone();
316 new_job.is_group_root = job.is_group_root;
317
318 let new_id = new_job.id.clone();
319 drop(jobs);
320
321 self.add_job(new_job).await;
323 return Some(new_id);
324 }
325 _ => return None,
326 }
327 }
328 None
329 }
330
331 async fn emit_update(&self, job: &TransferJob) {
332 if let Some(app) = self.app_handle.read().await.as_ref() {
333 let event = TransferEvent {
334 job_id: job.id.clone(),
335 bytes_per_second: job.bytes_per_second,
336 processed_bytes: job.processed_bytes,
337 total_bytes: job.total_bytes,
338 status: job.status.clone(),
339 finished_at: job.finished_at,
340 };
341 let _ = app.emit("transfer-update", event);
342 }
343 }
344
345 fn is_multipart_upload_job(job: &TransferJob) -> bool {
346 if !matches!(job.transfer_type, TransferType::Upload) {
347 return false;
348 }
349
350 job.total_bytes >= MULTIPART_UPLOAD_THRESHOLD
351 || std::fs::metadata(&job.local_path)
352 .map(|metadata| metadata.len() >= MULTIPART_UPLOAD_THRESHOLD)
353 .unwrap_or(false)
354 }
355
356 async fn ensure_multipart_job_active(&self, id: &str) -> crate::error::Result<()> {
357 match self.get_job(id).await.map(|job| job.status) {
358 Some(TransferStatus::InProgress) => Ok(()),
359 Some(TransferStatus::Cancelled) | None => Err(crate::error::AppError::ConfigError(
360 "Multipart upload was cancelled".to_string(),
361 )),
362 Some(status) => Err(crate::error::AppError::ConfigError(format!(
363 "Multipart upload stopped because the transfer is no longer active ({status:?})"
364 ))),
365 }
366 }
367
368 async fn acquire_slot(&self) -> ActiveSlotGuard {
369 loop {
370 let notified = self.slot_notify.notified();
371 tokio::pin!(notified);
372 notified.as_mut().enable();
373 let max = self.max_concurrency.load(Ordering::Acquire).max(1);
374 let active = self.active_count.load(Ordering::Acquire);
375
376 if active < max {
377 if self
378 .active_count
379 .compare_exchange_weak(active, active + 1, Ordering::AcqRel, Ordering::Acquire)
380 .is_ok()
381 {
382 return ActiveSlotGuard {
383 active_count: self.active_count.clone(),
384 slot_notify: self.slot_notify.clone(),
385 };
386 }
387 continue;
388 }
389
390 notified.await;
391 }
392 }
393
394 pub async fn process_queue(
396 self: Arc<Self>,
397 s3_manager: Arc<RwLock<S3ClientManager>>,
398 profile_manager: Arc<RwLock<ProfileManager>>,
399 ) {
400 let manager = self.clone();
401
402 tokio::spawn(async move {
403 loop {
404 let next_id = {
406 let mut queue = manager.queue.lock().await;
407 if queue.is_empty() {
408 break;
409 }
410 queue.pop_front().expect("queue checked above")
411 };
412
413 let slot_guard = manager.acquire_slot().await;
415
416 let manager_inner = manager.clone();
418 let s3_inner = s3_manager.clone();
419 let profiles_inner = profile_manager.clone();
420 let id_inner = next_id.clone();
421
422 let handle = tokio::spawn(async move {
423 let _slot_guard = slot_guard;
424 let should_run = matches!(
425 manager_inner.get_job(&id_inner).await.map(|job| job.status),
426 Some(TransferStatus::Pending)
427 );
428
429 if !should_run {
430 return;
431 }
432
433 manager_inner
435 .update_job_status(&id_inner, TransferStatus::InProgress)
436 .await;
437
438 let job_opt = manager_inner.get_job(&id_inner).await;
440 if let Some(job) = job_opt {
441 if job.status != TransferStatus::InProgress {
442 return;
443 }
444 let profile_result = {
445 let profiles = profiles_inner.read().await;
446 profiles.get_profile(&job.profile_id).await
447 };
448
449 let result = match profile_result {
450 Ok(profile) => {
451 let execution = manager_inner.execute_job(&job, s3_inner, &profile);
452 tokio::pin!(execution);
453 let mut timer = tokio::time::interval(Duration::from_millis(250));
454 timer.set_missed_tick_behavior(
455 tokio::time::MissedTickBehavior::Skip,
456 );
457 let mut speed =
458 super::speed::SpeedWindow::new(std::time::Instant::now());
459 loop {
460 tokio::select! {
461 result = &mut execution => break result,
462 _ = timer.tick() => {
463 let (position, attempt) = {
464 let progress = job.progress.lock().unwrap();
465 (progress.position, progress.attempt)
466 };
467 let rate = speed.sample(std::time::Instant::now(), position, attempt);
468 let mut jobs = manager_inner.jobs.write().await;
469 if let Some(current) = jobs.get_mut(&job.id) {
470 if current.status == TransferStatus::InProgress {
471 current.processed_bytes = if current.total_bytes > 0 {
472 position.min(current.total_bytes)
473 } else {
474 position
475 };
476 current.bytes_per_second = rate;
477 manager_inner.emit_update(current).await;
478 }
479 }
480 }
481 }
482 }
483 }
484 Err(err) => Err(err),
485 };
486
487 match result {
488 Ok(_) => {
489 if let Some(current_job) = manager_inner.get_job(&id_inner).await {
491 if !matches!(current_job.status, TransferStatus::Cancelled) {
492 manager_inner
493 .update_job_status(&id_inner, TransferStatus::Completed)
494 .await;
495 }
496 }
497 }
498 Err(e) => {
499 let was_cancelled = matches!(
500 manager_inner.get_job(&id_inner).await.map(|job| job.status),
501 Some(TransferStatus::Cancelled) | None
502 );
503 if !was_cancelled {
504 log::error!("Transfer job {} failed: {}", id_inner, e);
505 manager_inner
506 .update_job_status(
507 &id_inner,
508 TransferStatus::Failed(e.to_string()),
509 )
510 .await;
511 }
512 }
513 }
514 }
515
516 let mut handles = manager_inner.abort_handles.write().await;
518 handles.remove(&id_inner);
519 });
520
521 let mut handles = manager.abort_handles.write().await;
523 handles.insert(next_id, handle.abort_handle());
524 }
525 });
526 }
527
528 async fn update_job_status(&self, id: &str, status: TransferStatus) {
529 {
530 let mut jobs = self.jobs.write().await;
531 if let Some(job) = jobs.get_mut(id) {
532 if matches!(
533 job.status,
534 TransferStatus::Cancelled
535 | TransferStatus::Completed
536 | TransferStatus::Failed(_)
537 ) && job.status != status
538 {
539 return;
540 }
541 job.status = status.clone();
542 job.bytes_per_second = 0.0;
543 match status {
545 TransferStatus::Completed
546 | TransferStatus::Failed(_)
547 | TransferStatus::Cancelled => {
548 job.finished_at = Some(chrono::Utc::now().timestamp_millis());
549 }
550 _ => {}
551 }
552 }
553 }
554 self.persist_or_log().await;
555 if let Some(job) = self.get_job(id).await {
556 self.emit_update(&job).await;
557 }
558 }
559
560 async fn update_job_total_size(&self, id: &str, size: u64) {
561 {
562 let mut jobs = self.jobs.write().await;
563 if let Some(job) = jobs.get_mut(id) {
564 job.total_bytes = size;
565 }
566 }
567 if let Some(job) = self.get_job(id).await {
568 self.emit_update(&job).await;
569 }
570 }
571
572 async fn update_job_progress(&self, id: &str, processed: u64) {
573 {
574 let mut jobs = self.jobs.write().await;
575 if let Some(job) = jobs.get_mut(id) {
576 if job.status != TransferStatus::InProgress {
577 return;
578 }
579 job.processed_bytes = processed;
580 job.progress.lock().unwrap().position = processed;
581 }
582 }
583 if let Some(job) = self.get_job(id).await {
584 self.emit_update(&job).await;
585 }
586 }
587
588 async fn upload_file(
589 &self,
590 client: &Client,
591 job: &TransferJob,
592 content_type: &str,
593 ) -> crate::error::Result<()> {
594 let metadata = tokio::fs::metadata(&job.local_path)
595 .await
596 .map_err(|error| {
597 crate::error::AppError::IoError(format!(
598 "Cannot read upload file '{}': {error}",
599 job.local_path
600 ))
601 })?;
602 if !metadata.is_file() {
603 return Err(crate::error::AppError::IoError(format!(
604 "Upload source '{}' is not a regular file",
605 job.local_path
606 )));
607 }
608
609 let file_size = metadata.len();
610 if job.total_bytes != file_size {
611 self.update_job_total_size(&job.id, file_size).await;
612 }
613
614 if file_size < MULTIPART_UPLOAD_THRESHOLD {
615 let body = ByteStream::from_path(&job.local_path)
616 .await
617 .map_err(|error| crate::error::AppError::IoError(error.to_string()))?;
618 let body = super::speed::track(body, job.progress.clone(), 0);
619 let timeout = transfer_timeout_for(file_size);
620 let send = client
621 .put_object()
622 .bucket(&job.bucket)
623 .key(&job.key)
624 .content_type(content_type)
625 .if_none_match("*")
626 .body(body)
627 .send();
628 match tokio::time::timeout(timeout, send).await {
629 Ok(Ok(_)) => {}
630 Ok(Err(error)) => {
631 return Err(crate::error::AppError::S3Error(format!(
632 "Single-part upload failed for s3://{}/{}: {}",
633 job.bucket,
634 job.key,
635 DisplayErrorContext(&error)
636 )));
637 }
638 Err(_) => {
639 return Err(crate::error::AppError::S3Error(format!(
640 "Single-part upload timed out after {}s for s3://{}/{} with no response from S3 \
641 (check the endpoint, region, network/proxy, and TLS configuration)",
642 timeout.as_secs(),
643 job.bucket,
644 job.key
645 )));
646 }
647 }
648 self.update_job_progress(&job.id, file_size).await;
649 return Ok(());
650 }
651
652 self.multipart_upload_file(client, job, content_type, file_size)
653 .await
654 }
655
656 async fn multipart_upload_file(
657 &self,
658 client: &Client,
659 job: &TransferJob,
660 content_type: &str,
661 file_size: u64,
662 ) -> crate::error::Result<()> {
663 self.ensure_multipart_job_active(&job.id).await?;
664 let plan = plan_multipart_upload(file_size)?;
665 let create_send = client
666 .create_multipart_upload()
667 .bucket(&job.bucket)
668 .key(&job.key)
669 .content_type(content_type)
670 .send();
671 let created = match tokio::time::timeout(METADATA_REQUEST_TIMEOUT, create_send).await {
672 Ok(Ok(output)) => output,
673 Ok(Err(error)) => {
674 return Err(crate::error::AppError::S3Error(format!(
675 "Could not start multipart upload for s3://{}/{}: {}",
676 job.bucket,
677 job.key,
678 DisplayErrorContext(&error)
679 )));
680 }
681 Err(_) => {
682 return Err(crate::error::AppError::S3Error(format!(
683 "Starting multipart upload timed out after {}s for s3://{}/{} with no response from S3 \
684 (check the endpoint, region, network/proxy, and TLS configuration)",
685 METADATA_REQUEST_TIMEOUT.as_secs(),
686 job.bucket,
687 job.key
688 )));
689 }
690 };
691 let upload_id = created.upload_id().ok_or_else(|| {
692 crate::error::AppError::S3Error(format!(
693 "S3 did not return an upload ID for s3://{}/{}",
694 job.bucket, job.key
695 ))
696 })?;
697 let mut guard =
698 MultipartUploadGuard::new(client.clone(), &job.bucket, &job.key, upload_id.to_string());
699
700 let upload_result: crate::error::Result<()> = async {
701 self.ensure_multipart_job_active(&job.id).await?;
702 let mut completed_parts = Vec::with_capacity(plan.parts.len());
703 let mut uploaded_bytes = 0_u64;
704
705 for part in &plan.parts {
706 self.ensure_multipart_job_active(&job.id).await?;
707 let body = ByteStream::read_from()
708 .path(&job.local_path)
709 .offset(part.offset)
710 .length(Length::Exact(part.length))
711 .buffer_size(1024 * 1024)
712 .build()
713 .await
714 .map_err(|error| {
715 crate::error::AppError::IoError(format!(
716 "Could not read part {} from '{}': {error}",
717 part.number, job.local_path
718 ))
719 })?;
720
721 let body = super::speed::track(body, job.progress.clone(), part.offset);
722 let part_timeout = transfer_timeout_for(part.length);
723 let part_send = client
724 .upload_part()
725 .bucket(&job.bucket)
726 .key(&job.key)
727 .upload_id(guard.upload_id())
728 .part_number(part.number)
729 .content_length(part.length as i64)
730 .body(body)
731 .send();
732 let output = match tokio::time::timeout(part_timeout, part_send).await {
733 Ok(Ok(output)) => output,
734 Ok(Err(error)) => {
735 return Err(crate::error::AppError::S3Error(format!(
736 "Multipart upload failed at part {} of {} for s3://{}/{}: {}",
737 part.number,
738 plan.parts.len(),
739 job.bucket,
740 job.key,
741 DisplayErrorContext(&error)
742 )));
743 }
744 Err(_) => {
745 return Err(crate::error::AppError::S3Error(format!(
746 "Multipart upload timed out after {}s at part {} of {} for s3://{}/{} with no \
747 response from S3 (check the endpoint, region, network/proxy, and TLS configuration)",
748 part_timeout.as_secs(),
749 part.number,
750 plan.parts.len(),
751 job.bucket,
752 job.key
753 )));
754 }
755 };
756 self.ensure_multipart_job_active(&job.id).await?;
757 let e_tag = output.e_tag().ok_or_else(|| {
758 crate::error::AppError::S3Error(format!(
759 "S3 did not return an ETag for multipart part {} of s3://{}/{}",
760 part.number, job.bucket, job.key
761 ))
762 })?;
763
764 completed_parts.push(
765 CompletedPart::builder()
766 .part_number(part.number)
767 .e_tag(e_tag)
768 .build(),
769 );
770 uploaded_bytes += part.length;
771 self.update_job_progress(&job.id, uploaded_bytes).await;
772 }
773
774 self.ensure_multipart_job_active(&job.id).await?;
775 let completed_upload = CompletedMultipartUpload::builder()
776 .set_parts(Some(completed_parts))
777 .build();
778 let complete_send = client
779 .complete_multipart_upload()
780 .if_none_match("*")
781 .bucket(&job.bucket)
782 .key(&job.key)
783 .upload_id(guard.upload_id())
784 .multipart_upload(completed_upload)
785 .send();
786 match tokio::time::timeout(METADATA_REQUEST_TIMEOUT, complete_send).await {
787 Ok(Ok(_)) => {}
788 Ok(Err(error)) => {
789 return Err(crate::error::AppError::S3Error(format!(
790 "Could not complete multipart upload for s3://{}/{}: {}",
791 job.bucket,
792 job.key,
793 DisplayErrorContext(&error)
794 )));
795 }
796 Err(_) => {
797 return Err(crate::error::AppError::S3Error(format!(
798 "Completing multipart upload timed out after {}s for s3://{}/{} with no response from S3 \
799 (check the endpoint, region, network/proxy, and TLS configuration)",
800 METADATA_REQUEST_TIMEOUT.as_secs(),
801 job.bucket,
802 job.key
803 )));
804 }
805 }
806
807 Ok(())
808 }
809 .await;
810
811 if let Err(error) = upload_result {
812 if let Err(abort_error) = guard.abort().await {
813 log::error!("{}", abort_error);
814 }
815 return Err(error);
816 }
817
818 guard.disarm();
819 Ok(())
820 }
821
822 async fn execute_job(
823 &self,
824 job: &TransferJob,
825 s3_manager: Arc<RwLock<S3ClientManager>>,
826 profile: &Profile,
827 ) -> crate::error::Result<()> {
828 let resolved_region = {
829 let s3 = s3_manager.read().await;
830 s3.get_bucket_region(profile, &job.bucket)
831 }
832 .or(job.bucket_region.clone());
833
834 let client = {
835 let mut s3 = s3_manager.write().await;
836 let c = if let Some(ref region) = resolved_region {
837 s3.get_client_for_region(profile, region).await?
838 } else {
839 s3.get_client(profile).await?
840 };
841 c.clone()
842 };
843
844 let detect_region = async {
845 let retry_client = {
846 let mut s3 = s3_manager.write().await;
847 s3.get_client(profile).await?.clone()
848 };
849
850 let new_region = crate::s3::get_bucket_region(&retry_client, &job.bucket)
851 .await
852 .ok();
853 if let Some(ref region) = new_region {
854 let mut s3 = s3_manager.write().await;
855 s3.set_bucket_region(profile, &job.bucket, region.clone());
856 }
857 Ok::<Option<String>, crate::error::AppError>(new_region)
858 };
859
860 match job.transfer_type {
861 TransferType::Upload => {
862 let content_type = crate::s3::infer_content_type(&job.key);
863 if let Err(first_error) = self.upload_file(&client, job, &content_type).await {
864 if !matches!(
865 self.get_job(&job.id).await.map(|current| current.status),
866 Some(TransferStatus::InProgress)
867 ) {
868 return Err(first_error);
869 }
870 log::warn!(
871 "Upload failed using configured region {:?}; checking the bucket region once before returning the error: {}",
872 resolved_region,
873 first_error
874 );
875
876 let detected_region = detect_region.await?;
877 let retry_region = detected_region
878 .filter(|region| resolved_region.as_deref() != Some(region.as_str()));
879 if let Some(new_region) = retry_region {
880 let retry_client = {
881 let mut s3 = s3_manager.write().await;
882 s3.get_client_for_region(profile, &new_region)
883 .await?
884 .clone()
885 };
886 self.update_job_progress(&job.id, 0).await;
887 self.upload_file(&retry_client, job, &content_type)
888 .await
889 .map_err(|error| {
890 crate::error::AppError::S3Error(format!(
891 "Upload retry in detected bucket region '{new_region}' failed: {error}"
892 ))
893 })?;
894 } else {
895 return Err(first_error);
896 }
897 }
898
899 {
900 let mut s3 = s3_manager.write().await;
901 s3.remove_bucket_cache(&profile.id, &job.bucket);
902 }
903 }
904 TransferType::Download => {
905 let result = tokio::time::timeout(
906 METADATA_REQUEST_TIMEOUT,
907 client.get_object().bucket(&job.bucket).key(&job.key).send(),
908 )
909 .await;
910
911 let mut output = match result {
912 Ok(Ok(output)) => output,
913 Err(_) => {
914 return Err(crate::error::AppError::S3Error(format!(
915 "Download timed out after {}s for s3://{}/{} with no response from S3 \
916 (check the endpoint, region, network/proxy, and TLS configuration)",
917 METADATA_REQUEST_TIMEOUT.as_secs(),
918 job.bucket,
919 job.key
920 )));
921 }
922 Ok(Err(err)) => {
923 log::warn!(
924 "download transfer failed, attempting region discovery: {}",
925 DisplayErrorContext(&err)
926 );
927
928 if let Some(new_region) = detect_region.await? {
929 let retry_client = {
930 let mut s3 = s3_manager.write().await;
931 s3.get_client_for_region(profile, &new_region)
932 .await?
933 .clone()
934 };
935
936 tokio::time::timeout(
937 METADATA_REQUEST_TIMEOUT,
938 retry_client
939 .get_object()
940 .bucket(&job.bucket)
941 .key(&job.key)
942 .send(),
943 )
944 .await
945 .map_err(|_| {
946 crate::error::AppError::S3Error(format!(
947 "Retry download timed out after {}s for s3://{}/{} with no response \
948 from S3 (check the endpoint, region, network/proxy, and TLS configuration)",
949 METADATA_REQUEST_TIMEOUT.as_secs(),
950 job.bucket,
951 job.key
952 ))
953 })?
954 .map_err(|e| {
955 crate::error::AppError::S3Error(format!(
956 "Retry download failed: {}",
957 DisplayErrorContext(&e)
958 ))
959 })?
960 } else {
961 return Err(crate::error::AppError::S3Error(format!(
962 "{}",
963 DisplayErrorContext(&err)
964 )));
965 }
966 }
967 };
968
969 let destination = job.download_destination.as_ref().ok_or_else(|| {
970 crate::error::AppError::IoError(
971 "Missing download destination. Queue the download again.".to_string(),
972 )
973 })?;
974 let mut download = destination.begin()?;
975
976 let mut downloaded: u64 = 0;
977 let mut last_update = std::time::Instant::now();
978
979 while let Some(bytes) =
980 tokio::time::timeout(METADATA_REQUEST_TIMEOUT, output.body.try_next())
981 .await
982 .map_err(|_| {
983 crate::error::AppError::S3Error(
984 "Download stalled for 60 seconds. Check the connection and retry."
985 .into(),
986 )
987 })?
988 .map_err(|e| {
989 crate::error::AppError::S3Error(format!("{}", DisplayErrorContext(&e)))
990 })?
991 {
992 download
993 .write_all(&bytes)
994 .await
995 .map_err(|e| crate::error::AppError::IoError(e.to_string()))?;
996
997 downloaded += bytes.len() as u64;
998
999 if last_update.elapsed() >= std::time::Duration::from_millis(100) {
1000 self.update_job_progress(&job.id, downloaded).await;
1001 last_update = std::time::Instant::now();
1002 }
1003 }
1004
1005 self.update_job_progress(&job.id, downloaded).await;
1006 if job.total_bytes == 0 {
1007 self.update_job_total_size(&job.id, downloaded).await;
1008 }
1009 download.finish_writing().await?;
1010 let mut jobs = self.jobs.write().await;
1011 let current = jobs.get_mut(&job.id).ok_or_else(|| {
1012 crate::error::AppError::IoError("Download was removed".to_string())
1013 })?;
1014 if current.status != TransferStatus::InProgress {
1015 return Err(crate::error::AppError::IoError(
1016 "Download was cancelled".to_string(),
1017 ));
1018 }
1019 download.commit()?;
1020 current.status = TransferStatus::Completed;
1021 current.finished_at = Some(chrono::Utc::now().timestamp_millis());
1022 drop(jobs);
1023 self.persist_or_log().await;
1024 }
1025 }
1026
1027 Ok(())
1028 }
1029}
1030
1031#[cfg(test)]
1032mod tests {
1033 use super::{Client, RwLock, S3ClientManager, TransferManager};
1034
1035 #[tokio::test]
1036 async fn interrupted_jobs_are_recovered_without_restarting_writes() {
1037 let directory = tempfile::tempdir().unwrap();
1038 let path = directory.path().join("transfers.json");
1039 let manager = TransferManager::new();
1040 manager.configure_journal(path.clone()).await.unwrap();
1041 let job = TransferJob::new(
1042 TransferType::Upload,
1043 "profile".into(),
1044 "bucket".into(),
1045 None,
1046 "key".into(),
1047 "/unused".into(),
1048 10,
1049 );
1050 let id = job.id.clone();
1051 manager.add_job(job).await;
1052 let restarted = TransferManager::new();
1053 restarted.configure_journal(path).await.unwrap();
1054 assert!(matches!(
1055 restarted.get_job(&id).await.unwrap().status,
1056 TransferStatus::Failed(_)
1057 ));
1058 assert!(restarted.queue.lock().await.is_empty());
1059 }
1060
1061 #[tokio::test]
1062 async fn corrupt_recovery_journal_is_preserved_without_blocking_startup() {
1063 let directory = tempfile::tempdir().unwrap();
1064 let path = directory.path().join("transfers.json");
1065 std::fs::write(&path, "broken").unwrap();
1066 let manager = TransferManager::new();
1067 manager.configure_journal(path).await.unwrap();
1068 assert!(manager.list_jobs().await.is_empty());
1069 let backup = std::fs::read_dir(directory.path())
1070 .unwrap()
1071 .map(|entry| entry.unwrap().path())
1072 .find(|path| {
1073 path.file_name()
1074 .unwrap()
1075 .to_string_lossy()
1076 .starts_with("transfers.invalid-")
1077 })
1078 .unwrap();
1079 assert_eq!(std::fs::read_to_string(backup).unwrap(), "broken");
1080 }
1081 use crate::credentials::{CredentialType, Profile};
1082 use crate::transfer::{TransferJob, TransferStatus, TransferType};
1083 use std::path::PathBuf;
1084 use std::sync::Arc;
1085 use std::time::Duration;
1086
1087 #[test]
1088 fn upload_timeouts_cover_small_and_large_payloads_without_overflow() {
1089 assert_eq!(super::transfer_timeout_for(0), Duration::from_secs(60));
1090 assert_eq!(
1091 super::transfer_timeout_for(10 * 1024 * 1024),
1092 Duration::from_secs(103)
1093 );
1094 assert_eq!(
1095 super::transfer_timeout_for(u64::MAX),
1096 Duration::from_secs(1800)
1097 );
1098 }
1099
1100 #[tokio::test]
1101 async fn stalled_download_releases_its_partial_file() {
1102 let (endpoint, received, server) = crate::commands::test_s3::stalled_endpoint(
1103 "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ndo",
1104 )
1105 .await;
1106 let profile = Profile::new(
1107 "test".into(),
1108 CredentialType::CustomEndpoint {
1109 endpoint_url: endpoint,
1110 access_key_id: "TEST".into(),
1111 secret_access_key: "test-secret".into(),
1112 },
1113 Some("us-east-1".into()),
1114 );
1115 let directory = tempfile::tempdir().unwrap();
1116 let path = directory.path().join("result.txt");
1117 let mut job = test_job(TransferStatus::InProgress);
1118 job.profile_id = profile.id.clone();
1119 job.local_path = path.to_string_lossy().into_owned();
1120 job.download_destination = Some(Arc::new(
1121 crate::transfer::download::DownloadDestination::from_path(&path, false).unwrap(),
1122 ));
1123 let manager = TransferManager::new();
1124 manager.add_job(job.clone()).await;
1125 let task = tokio::spawn(async move {
1126 manager
1127 .execute_job(
1128 &job,
1129 Arc::new(RwLock::new(S3ClientManager::new())),
1130 &profile,
1131 )
1132 .await
1133 });
1134 received.await.unwrap();
1135 tokio::time::pause();
1136 let result = task.await.unwrap();
1137 tokio::time::resume();
1138 server.abort();
1139 assert!(result.is_err());
1140 assert!(!path.exists());
1141 assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0);
1142 }
1143
1144 #[tokio::test]
1145 async fn failed_upload_retains_the_provider_error() {
1146 let (client, server) =
1147 crate::commands::test_s3::scripted_client(vec![crate::commands::test_s3::response(
1148 403,
1149 "",
1150 "<Error><Code>AccessDenied</Code><Message>Uploads are disabled</Message></Error>",
1151 )])
1152 .await;
1153 let client = Client::from_conf(
1154 client
1155 .config()
1156 .to_builder()
1157 .request_checksum_calculation(
1158 aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired,
1159 )
1160 .build(),
1161 );
1162 let directory = tempfile::tempdir().unwrap();
1163 let path = directory.path().join("input.txt");
1164 std::fs::write(&path, b"test").unwrap();
1165 let mut job = test_job(TransferStatus::InProgress);
1166 job.local_path = path.to_string_lossy().into_owned();
1167 let error = TransferManager::new()
1168 .upload_file(&client, &job, "text/plain")
1169 .await
1170 .unwrap_err();
1171 assert!(error.to_string().contains("AccessDenied"));
1172 assert!(error.to_string().contains("Uploads are disabled"));
1173 assert_eq!(server.await.unwrap().len(), 1);
1174 }
1175
1176 #[tokio::test]
1177 async fn retried_download_retains_its_destination_and_profile() {
1178 let directory = tempfile::tempdir().unwrap();
1179 let path = directory.path().join("result.txt");
1180 let destination = Arc::new(
1181 crate::transfer::download::DownloadDestination::from_path(&path, false).unwrap(),
1182 );
1183 let mut job = TransferJob::new(
1184 TransferType::Download,
1185 "profile-a".into(),
1186 "bucket".into(),
1187 Some("region".into()),
1188 "folder/result.txt".into(),
1189 path.clone(),
1190 4,
1191 )
1192 .with_group("group".into(), "folder".into());
1193 job.download_destination = Some(destination);
1194 job.status = TransferStatus::Failed("Connection lost".into());
1195 job.processed_bytes = 2;
1196 let manager = TransferManager::new();
1197 manager.add_job(job.clone()).await;
1198 let retry_id = manager.retry_job(&job.id).await.unwrap();
1199 let retry = manager.get_job(&retry_id).await.unwrap();
1200 assert_ne!(retry_id, job.id);
1201 assert_eq!(retry.profile_id, "profile-a");
1202 assert_eq!(retry.parent_group_id, job.parent_group_id);
1203 assert_eq!(retry.group_name, job.group_name);
1204 assert_eq!(retry.status, TransferStatus::Pending);
1205 assert_eq!(retry.processed_bytes, 0);
1206 assert!(matches!(
1207 manager.get_job(&job.id).await.unwrap().status,
1208 TransferStatus::Failed(_)
1209 ));
1210 let mut pending = retry
1211 .download_destination
1212 .expect("A retry must retain the selected download directory")
1213 .begin()
1214 .unwrap();
1215 pending.write_all(b"done").await.unwrap();
1216 pending.finish_writing().await.unwrap();
1217 pending.commit().unwrap();
1218 assert_eq!(std::fs::read(path).unwrap(), b"done");
1219 }
1220
1221 fn test_job(status: TransferStatus) -> TransferJob {
1222 let mut job = TransferJob::new(
1223 TransferType::Download,
1224 "profile-a".into(),
1225 "bucket".into(),
1226 None,
1227 "file.txt".into(),
1228 PathBuf::from("file.txt"),
1229 10,
1230 );
1231 job.status = status;
1232 job
1233 }
1234
1235 #[tokio::test]
1236 async fn terminal_jobs_clear_speed_and_ignore_late_progress() {
1237 for status in [
1238 TransferStatus::Completed,
1239 TransferStatus::Failed("offline".into()),
1240 TransferStatus::Cancelled,
1241 ] {
1242 let manager = TransferManager::new();
1243 let mut job = test_job(TransferStatus::InProgress);
1244 job.bytes_per_second = 1024.0;
1245 manager.add_job(job.clone()).await;
1246 if status == TransferStatus::Cancelled {
1247 manager.cancel_job(&job.id).await;
1248 } else {
1249 manager.update_job_status(&job.id, status.clone()).await;
1250 }
1251 manager.update_job_progress(&job.id, 5).await;
1252 let saved = manager.get_job(&job.id).await.unwrap();
1253 assert_eq!(saved.status, status);
1254 assert_eq!(saved.bytes_per_second, 0.0);
1255 assert_eq!(saved.processed_bytes, 0);
1256 }
1257 }
1258
1259 #[tokio::test]
1260 async fn cancelling_pending_work_keeps_history_and_removes_it_from_the_queue() {
1261 let manager = TransferManager::new();
1262 let job = test_job(TransferStatus::Pending);
1263 manager.add_job(job.clone()).await;
1264 assert!(manager.cancel_job(&job.id).await);
1265 assert_eq!(
1266 manager.get_job(&job.id).await.unwrap().status,
1267 TransferStatus::Cancelled
1268 );
1269 assert!(!manager.queue.lock().await.contains(&job.id));
1270 assert!(!manager.cancel_job(&job.id).await);
1271 assert!(!manager.cancel_job("missing").await);
1272 }
1273
1274 #[tokio::test]
1275 async fn cancelling_active_downloads_aborts_the_task_and_releases_capacity() {
1276 let manager = TransferManager::new();
1277 let job = test_job(TransferStatus::InProgress);
1278 manager.add_job(job.clone()).await;
1279 let slot = manager.acquire_slot().await;
1280 let task = tokio::spawn(async move {
1281 let _slot = slot;
1282 std::future::pending::<()>().await;
1283 });
1284 manager
1285 .abort_handles
1286 .write()
1287 .await
1288 .insert(job.id.clone(), task.abort_handle());
1289 assert!(manager.cancel_job(&job.id).await);
1290 assert!(tokio::time::timeout(Duration::from_secs(2), task)
1291 .await
1292 .unwrap()
1293 .unwrap_err()
1294 .is_cancelled());
1295 assert_eq!(
1296 manager
1297 .active_count
1298 .load(std::sync::atomic::Ordering::Acquire),
1299 0
1300 );
1301 }
1302
1303 #[tokio::test]
1304 async fn clearing_history_retains_pending_and_active_transfers() {
1305 let manager = TransferManager::new();
1306 let pending = test_job(TransferStatus::Pending);
1307 let active = test_job(TransferStatus::InProgress);
1308 for job in [
1309 pending.clone(),
1310 active.clone(),
1311 test_job(TransferStatus::Completed),
1312 test_job(TransferStatus::Failed("offline".into())),
1313 test_job(TransferStatus::Cancelled),
1314 ] {
1315 manager.add_job(job).await;
1316 }
1317 assert_eq!(manager.clear_completed().await, 3);
1318 let jobs = manager.list_jobs().await;
1319 assert_eq!(jobs.len(), 2);
1320 assert!(jobs.iter().any(|job| job.id == pending.id));
1321 assert!(jobs.iter().any(|job| job.id == active.id));
1322 assert_eq!(manager.clear_completed().await, 0);
1323 }
1324
1325 #[tokio::test]
1326 async fn retry_refuses_active_and_completed_work() {
1327 let manager = TransferManager::new();
1328 for status in [
1329 TransferStatus::Pending,
1330 TransferStatus::InProgress,
1331 TransferStatus::Completed,
1332 ] {
1333 let job = test_job(status);
1334 manager.add_job(job.clone()).await;
1335 assert!(manager.retry_job(&job.id).await.is_none());
1336 }
1337 assert!(manager.retry_job("missing").await.is_none());
1338 assert_eq!(manager.list_jobs().await.len(), 3);
1339 }
1340
1341 #[tokio::test]
1342 async fn removing_pending_work_removes_both_its_queue_entry_and_history() {
1343 let manager = TransferManager::new();
1344 let job = test_job(TransferStatus::Pending);
1345 manager.add_job(job.clone()).await;
1346 assert!(manager.remove_job(&job.id).await);
1347 assert!(manager.get_job(&job.id).await.is_none());
1348 assert!(!manager.queue.lock().await.contains(&job.id));
1349 assert!(!manager.remove_job(&job.id).await);
1350 }
1351
1352 #[tokio::test]
1353 async fn transfer_slots_follow_concurrency_changes_without_stalling() {
1354 let manager = TransferManager::new();
1355 manager.set_max_concurrency(1);
1356 let first = manager.acquire_slot().await;
1357 let second = manager.acquire_slot();
1358 tokio::pin!(second);
1359 assert!(futures::poll!(second.as_mut()).is_pending());
1360 manager.set_max_concurrency(2);
1361 let second = tokio::time::timeout(Duration::from_secs(1), second)
1362 .await
1363 .unwrap();
1364 manager.set_max_concurrency(1);
1365 let third = manager.acquire_slot();
1366 tokio::pin!(third);
1367 assert!(futures::poll!(third.as_mut()).is_pending());
1368 drop(second);
1369 assert!(futures::poll!(third.as_mut()).is_pending());
1370 drop(first);
1371 tokio::time::timeout(Duration::from_secs(1), third)
1372 .await
1373 .unwrap();
1374 }
1375
1376 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1377 async fn released_slots_allow_all_waiting_transfers_to_finish() {
1378 use std::sync::atomic::{AtomicUsize, Ordering};
1379 let manager = Arc::new(TransferManager::new());
1380 manager.set_max_concurrency(3);
1381 let peak = Arc::new(AtomicUsize::new(0));
1382 let tasks = (0..200)
1383 .map(|_| {
1384 let manager = manager.clone();
1385 let peak = peak.clone();
1386 tokio::spawn(async move {
1387 let _slot = manager.acquire_slot().await;
1388 peak.fetch_max(
1389 manager.active_count.load(Ordering::Acquire),
1390 Ordering::AcqRel,
1391 );
1392 tokio::task::yield_now().await;
1393 })
1394 })
1395 .collect::<Vec<_>>();
1396 let results =
1397 tokio::time::timeout(Duration::from_secs(5), futures::future::join_all(tasks))
1398 .await
1399 .unwrap();
1400 assert!(results.into_iter().all(|result| result.is_ok()));
1401 assert!(peak.load(Ordering::Acquire) <= 3);
1402 assert_eq!(manager.active_count.load(Ordering::Acquire), 0);
1403 }
1404
1405 #[tokio::test]
1409 #[ignore = "requires an S3-compatible integration-test endpoint"]
1410 async fn multipart_upload_round_trips_against_s3_endpoint() {
1411 let endpoint = std::env::var("BROWS3_S3_TEST_ENDPOINT")
1412 .expect("BROWS3_S3_TEST_ENDPOINT must identify the test S3 endpoint");
1413 let access_key =
1414 std::env::var("BROWS3_S3_TEST_ACCESS_KEY").unwrap_or_else(|_| "minioadmin".into());
1415 let secret_key =
1416 std::env::var("BROWS3_S3_TEST_SECRET_KEY").unwrap_or_else(|_| "minioadmin".into());
1417 let profile = Profile::new(
1418 "multipart-integration".to_string(),
1419 CredentialType::CustomEndpoint {
1420 endpoint_url: endpoint,
1421 access_key_id: access_key,
1422 secret_access_key: secret_key,
1423 },
1424 Some("us-east-1".to_string()),
1425 );
1426 let sdk_config = crate::s3::client::load_sdk_config(&profile, None).await;
1427 let client = crate::s3::client::client_from_sdk_config(&sdk_config, &profile);
1428 let bucket = format!("brows3-multipart-test-{}", uuid::Uuid::new_v4().simple());
1429 client
1430 .create_bucket()
1431 .bucket(&bucket)
1432 .send()
1433 .await
1434 .expect("test bucket should be created");
1435
1436 let path: PathBuf = std::env::temp_dir().join(format!(
1437 "brows3-multipart-{}.bin",
1438 uuid::Uuid::new_v4().simple()
1439 ));
1440 let file = tokio::fs::File::create(&path)
1441 .await
1442 .expect("sparse test file should be created");
1443 let size = 129 * 1024 * 1024;
1444 file.set_len(size)
1445 .await
1446 .expect("sparse test file should be sized");
1447 drop(file);
1448
1449 let manager = Arc::new(TransferManager::new());
1450 let job = TransferJob::new(
1451 TransferType::Upload,
1452 profile.id.clone(),
1453 bucket.clone(),
1454 profile.region.clone(),
1455 "large/test.bin".to_string(),
1456 path.clone(),
1457 size,
1458 );
1459 manager.add_job(job.clone()).await;
1460 manager
1461 .update_job_status(&job.id, TransferStatus::InProgress)
1462 .await;
1463 manager
1464 .upload_file(&client, &job, "application/octet-stream")
1465 .await
1466 .expect("multipart upload should complete");
1467
1468 let head = client
1469 .head_object()
1470 .bucket(&bucket)
1471 .key(&job.key)
1472 .send()
1473 .await
1474 .expect("uploaded object should be readable");
1475 assert_eq!(head.content_length(), Some(size as i64));
1476 assert_eq!(head.content_type(), Some("application/octet-stream"));
1477 assert!(head.e_tag().is_some_and(|e_tag| e_tag.contains('-')));
1478
1479 let incomplete = client
1480 .list_multipart_uploads()
1481 .bucket(&bucket)
1482 .send()
1483 .await
1484 .expect("multipart uploads should be listable");
1485 assert!(incomplete.uploads().is_empty());
1486
1487 let cancel_path: PathBuf = std::env::temp_dir().join(format!(
1488 "brows3-multipart-cancel-{}.bin",
1489 uuid::Uuid::new_v4().simple()
1490 ));
1491 let cancel_file = tokio::fs::File::create(&cancel_path)
1492 .await
1493 .expect("sparse cancellation file should be created");
1494 let cancel_size = 2 * 1024 * 1024 * 1024;
1495 cancel_file
1496 .set_len(cancel_size)
1497 .await
1498 .expect("sparse cancellation file should be sized");
1499 drop(cancel_file);
1500 let cancel_job = TransferJob::new(
1501 TransferType::Upload,
1502 profile.id.clone(),
1503 bucket.clone(),
1504 profile.region.clone(),
1505 "large/cancelled.bin".to_string(),
1506 cancel_path.clone(),
1507 cancel_size,
1508 );
1509 manager.add_job(cancel_job.clone()).await;
1510 manager
1511 .update_job_status(&cancel_job.id, TransferStatus::InProgress)
1512 .await;
1513
1514 let upload_task = {
1515 let task_manager = manager.clone();
1516 let task_client = client.clone();
1517 let task_job = cancel_job.clone();
1518 tokio::spawn(async move {
1519 task_manager
1520 .multipart_upload_file(
1521 &task_client,
1522 &task_job,
1523 "application/octet-stream",
1524 cancel_size,
1525 )
1526 .await
1527 })
1528 };
1529
1530 let mut upload_started = false;
1531 for _ in 0..100 {
1532 let active = client
1533 .list_multipart_uploads()
1534 .bucket(&bucket)
1535 .send()
1536 .await
1537 .expect("multipart uploads should be listable");
1538 if !active.uploads().is_empty() {
1539 upload_started = true;
1540 break;
1541 }
1542 tokio::time::sleep(Duration::from_millis(20)).await;
1543 }
1544 assert!(upload_started, "test multipart upload should start");
1545 assert!(manager.cancel_job(&cancel_job.id).await);
1546 let cancelled = tokio::time::timeout(Duration::from_secs(30), upload_task)
1547 .await
1548 .expect("cancelled multipart upload should finish cleanup")
1549 .expect("multipart task should not panic");
1550 assert!(cancelled.is_err());
1551 let incomplete = client
1552 .list_multipart_uploads()
1553 .bucket(&bucket)
1554 .send()
1555 .await
1556 .expect("multipart uploads should be listable after cancellation");
1557 assert!(incomplete.uploads().is_empty());
1558 assert!(client
1559 .head_object()
1560 .bucket(&bucket)
1561 .key(&cancel_job.key)
1562 .send()
1563 .await
1564 .is_err());
1565
1566 client
1567 .delete_object()
1568 .bucket(&bucket)
1569 .key(&job.key)
1570 .send()
1571 .await
1572 .expect("test object should be deleted");
1573 client
1574 .delete_bucket()
1575 .bucket(&bucket)
1576 .send()
1577 .await
1578 .expect("test bucket should be deleted");
1579 tokio::fs::remove_file(path)
1580 .await
1581 .expect("sparse test file should be deleted");
1582 tokio::fs::remove_file(cancel_path)
1583 .await
1584 .expect("sparse cancellation file should be deleted");
1585 }
1586}