Skip to main content

brows3_lib/transfer/
manager.rs

1use super::{TransferEvent, TransferJob, TransferStatus, TransferType};
2use crate::credentials::{Profile, ProfileManager};
3use crate::s3::{
4    plan_multipart_upload, MultipartUploadGuard, S3ClientManager, MULTIPART_UPLOAD_THRESHOLD,
5};
6use aws_sdk_s3::error::DisplayErrorContext;
7use aws_sdk_s3::primitives::ByteStream;
8use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
9use aws_sdk_s3::Client;
10use aws_smithy_types::byte_stream::Length;
11use std::collections::{HashMap, VecDeque};
12use std::sync::atomic::{AtomicUsize, Ordering};
13use std::sync::Arc;
14use std::time::Duration;
15use tauri::{AppHandle, Emitter};
16use tokio::sync::{Mutex, Notify, RwLock};
17
18const MIN_TRANSFER_TIMEOUT: Duration = Duration::from_secs(60);
19const MAX_TRANSFER_TIMEOUT: Duration = Duration::from_secs(30 * 60);
20const MIN_ASSUMED_THROUGHPUT_BYTES_PER_SEC: u64 = 100 * 1024;
21const METADATA_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
22
23fn transfer_timeout_for(size_bytes: u64) -> Duration {
24    let scaled_secs = size_bytes.div_ceil(MIN_ASSUMED_THROUGHPUT_BYTES_PER_SEC);
25    Duration::from_secs(scaled_secs).clamp(MIN_TRANSFER_TIMEOUT, MAX_TRANSFER_TIMEOUT)
26}
27
28// Define a safe shared state for the manager
29pub struct TransferManager {
30    jobs: Arc<RwLock<HashMap<String, TransferJob>>>,
31    queue: Arc<Mutex<VecDeque<String>>>, // List of Job IDs
32    abort_handles: Arc<RwLock<HashMap<String, tokio::task::AbortHandle>>>,
33    max_concurrency: Arc<AtomicUsize>,
34    active_count: Arc<AtomicUsize>,
35    slot_notify: Arc<Notify>,
36    app_handle: Arc<RwLock<Option<AppHandle>>>,
37    journal: Mutex<Option<std::path::PathBuf>>,
38}
39
40struct ActiveSlotGuard {
41    active_count: Arc<AtomicUsize>,
42    slot_notify: Arc<Notify>,
43}
44
45impl Drop for ActiveSlotGuard {
46    fn drop(&mut self) {
47        self.active_count.fetch_sub(1, Ordering::AcqRel);
48        self.slot_notify.notify_waiters();
49    }
50}
51
52impl Default for TransferManager {
53    fn default() -> Self {
54        Self::new()
55    }
56}
57
58impl TransferManager {
59    pub fn new() -> Self {
60        Self {
61            jobs: Arc::new(RwLock::new(HashMap::new())),
62            queue: Arc::new(Mutex::new(VecDeque::new())),
63            abort_handles: Arc::new(RwLock::new(HashMap::new())),
64            max_concurrency: Arc::new(AtomicUsize::new(5)),
65            active_count: Arc::new(AtomicUsize::new(0)),
66            slot_notify: Arc::new(Notify::new()),
67            app_handle: Arc::new(RwLock::new(None)),
68            journal: Mutex::new(None),
69        }
70    }
71
72    pub async fn set_app_handle(&self, app_handle: AppHandle) {
73        let mut handle = self.app_handle.write().await;
74        *handle = Some(app_handle);
75    }
76
77    pub async fn configure_journal(&self, path: std::path::PathBuf) -> crate::error::Result<()> {
78        *self.journal.lock().await = Some(path.clone());
79        if path.exists() {
80            let content = std::fs::read(&path)?;
81            let mut restored: Vec<TransferJob> = match serde_json::from_slice(&content) {
82                Ok(jobs) => jobs,
83                Err(error) => {
84                    let backup = path
85                        .with_file_name(format!("transfers.invalid-{}.json", uuid::Uuid::new_v4()));
86                    std::fs::rename(&path, &backup)?;
87                    log::error!(
88                        "Invalid transfer journal preserved at {}: {error}",
89                        backup.display()
90                    );
91                    Vec::new()
92                }
93            };
94            for job in &mut restored {
95                job.bytes_per_second = 0.0;
96                if matches!(
97                    job.status,
98                    TransferStatus::Pending | TransferStatus::InProgress
99                ) {
100                    job.status = TransferStatus::Failed(match job.transfer_type {
101                        TransferType::Upload => "Interrupted when the app closed. Check the destination before retrying; existing objects will not be replaced.".into(),
102                        TransferType::Download => "Interrupted when the app closed. Queue the download again to choose its destination.".into(),
103                    });
104                    job.finished_at = Some(chrono::Utc::now().timestamp_millis());
105                }
106            }
107            *self.jobs.write().await = restored
108                .into_iter()
109                .map(|job| (job.id.clone(), job))
110                .collect();
111        }
112        self.persist_jobs().await
113    }
114
115    async fn persist_jobs(&self) -> crate::error::Result<()> {
116        // Serialize snapshots and writes together, so an older snapshot cannot win.
117        let journal = self.journal.lock().await;
118        let Some(path) = journal.as_ref() else {
119            return Ok(());
120        };
121        let mut jobs = self.jobs.write().await;
122        let mut finished: Vec<_> = jobs
123            .values()
124            .filter(|job| {
125                matches!(
126                    job.status,
127                    TransferStatus::Completed
128                        | TransferStatus::Failed(_)
129                        | TransferStatus::Cancelled
130                )
131            })
132            .map(|job| (job.finished_at.unwrap_or(job.created_at), job.id.clone()))
133            .collect();
134        finished.sort_unstable();
135        let excess = finished.len().saturating_sub(1000);
136        for (_, id) in finished.into_iter().take(excess) {
137            jobs.remove(&id);
138        }
139        let bytes = serde_json::to_vec(&jobs.values().collect::<Vec<_>>())?;
140        drop(jobs);
141        let path = path.clone();
142        tokio::task::spawn_blocking(move || crate::credentials::write_private_file(&path, &bytes))
143            .await
144            .map_err(|error| crate::error::AppError::IoError(error.to_string()))??;
145        Ok(())
146    }
147
148    async fn persist_or_log(&self) {
149        if let Err(error) = self.persist_jobs().await {
150            log::error!("Transfer recovery journal could not be saved: {error}");
151        }
152    }
153
154    pub async fn add_job(&self, job: TransferJob) {
155        self.add_jobs(vec![job]).await;
156    }
157
158    pub async fn add_jobs(&self, added: Vec<TransferJob>) {
159        {
160            let mut jobs = self.jobs.write().await;
161            for job in &added {
162                jobs.insert(job.id.clone(), job.clone());
163            }
164        }
165        if let Err(error) = self.persist_jobs().await {
166            let mut jobs = self.jobs.write().await;
167            for job in &added {
168                if let Some(current) = jobs.get_mut(&job.id) {
169                    current.status =
170                        TransferStatus::Failed(format!("Could not save queued transfer: {error}"));
171                }
172            }
173            return;
174        }
175        {
176            let mut queue = self.queue.lock().await;
177            queue.extend(added.iter().map(|job| job.id.clone()));
178        }
179        for job in added {
180            if let Some(app) = self.app_handle.read().await.as_ref() {
181                let _ = app.emit("transfer-added", &job);
182            }
183            self.emit_update(&job).await;
184        }
185    }
186
187    pub fn set_max_concurrency(&self, max: usize) {
188        let clamped = max.clamp(1, 20);
189        self.max_concurrency.store(clamped, Ordering::Release);
190        self.slot_notify.notify_waiters();
191    }
192
193    pub async fn get_job(&self, id: &str) -> Option<TransferJob> {
194        let jobs = self.jobs.read().await;
195        jobs.get(id).cloned()
196    }
197
198    pub async fn list_jobs(&self) -> Vec<TransferJob> {
199        let jobs = self.jobs.read().await;
200        let mut list: Vec<TransferJob> = jobs.values().cloned().collect();
201        list.sort_by_key(|job| std::cmp::Reverse(job.created_at)); // Newest first
202        list
203    }
204
205    /// Cancel a transfer job
206    pub async fn cancel_job(&self, id: &str) -> bool {
207        let mut jobs = self.jobs.write().await;
208        if let Some(job) = jobs.get_mut(id) {
209            // Can only cancel Pending or InProgress jobs
210            match job.status {
211                TransferStatus::Pending | TransferStatus::InProgress => {
212                    job.status = TransferStatus::Cancelled;
213                    job.bytes_per_second = 0.0;
214                    let job_clone = job.clone();
215                    let needs_graceful_cleanup = Self::is_multipart_upload_job(job);
216
217                    {
218                        let mut queue = self.queue.lock().await;
219                        queue.retain(|job_id| job_id != id);
220                    }
221
222                    // Multipart uploads finish their current bounded part and
223                    // abort by upload ID. Other transfers can stop immediately.
224                    let mut handles = self.abort_handles.write().await;
225                    if let Some(handle) = handles.remove(id) {
226                        if needs_graceful_cleanup {
227                            log::info!(
228                                "Cancellation requested for multipart job {}; waiting for the current part so the server-side upload can be aborted cleanly",
229                                id
230                            );
231                        } else {
232                            handle.abort();
233                            log::info!("Aborted job task: {}", id);
234                        }
235                    }
236
237                    drop(handles);
238                    drop(jobs);
239                    self.persist_or_log().await;
240                    self.emit_update(&job_clone).await;
241                    return true;
242                }
243                _ => return false,
244            }
245        }
246        false
247    }
248
249    /// Remove a specific transfer job from history
250    pub async fn remove_job(&self, id: &str) -> bool {
251        let needs_graceful_cleanup = self
252            .get_job(id)
253            .await
254            .as_ref()
255            .is_some_and(Self::is_multipart_upload_job);
256
257        {
258            let mut queue = self.queue.lock().await;
259            queue.retain(|job_id| job_id != id);
260        }
261
262        let mut handles = self.abort_handles.write().await;
263        if let Some(handle) = handles.remove(id) {
264            if !needs_graceful_cleanup {
265                handle.abort();
266            }
267        }
268        drop(handles);
269
270        let mut jobs = self.jobs.write().await;
271        let removed = jobs.remove(id).is_some();
272        drop(jobs);
273        self.persist_or_log().await;
274        removed
275    }
276
277    /// Clear all completed/failed/cancelled transfers
278    pub async fn clear_completed(&self) -> usize {
279        let mut jobs = self.jobs.write().await;
280        let initial_count = jobs.len();
281        jobs.retain(|_, job| {
282            matches!(
283                job.status,
284                TransferStatus::Pending | TransferStatus::InProgress
285            )
286        });
287        let removed = initial_count - jobs.len();
288        drop(jobs);
289        self.persist_or_log().await;
290        removed
291    }
292
293    /// Retry a failed transfer
294    pub async fn retry_job(&self, id: &str) -> Option<String> {
295        let jobs = self.jobs.read().await;
296        if let Some(job) = jobs.get(id) {
297            // Can only retry Failed or Cancelled jobs
298            match &job.status {
299                TransferStatus::Failed(_) | TransferStatus::Cancelled => {
300                    // Create a new job with same details
301                    let mut new_job = TransferJob::new(
302                        job.transfer_type.clone(),
303                        job.profile_id.clone(),
304                        job.bucket.clone(),
305                        job.bucket_region.clone(),
306                        job.key.clone(),
307                        std::path::PathBuf::from(&job.local_path),
308                        job.total_bytes,
309                    );
310
311                    new_job.download_destination = job.download_destination.clone();
312
313                    // Preserve grouping info
314                    new_job.parent_group_id = job.parent_group_id.clone();
315                    new_job.group_name = job.group_name.clone();
316                    new_job.is_group_root = job.is_group_root;
317
318                    let new_id = new_job.id.clone();
319                    drop(jobs);
320
321                    // Add the new job to queue
322                    self.add_job(new_job).await;
323                    return Some(new_id);
324                }
325                _ => return None,
326            }
327        }
328        None
329    }
330
331    async fn emit_update(&self, job: &TransferJob) {
332        if let Some(app) = self.app_handle.read().await.as_ref() {
333            let event = TransferEvent {
334                job_id: job.id.clone(),
335                bytes_per_second: job.bytes_per_second,
336                processed_bytes: job.processed_bytes,
337                total_bytes: job.total_bytes,
338                status: job.status.clone(),
339                finished_at: job.finished_at,
340            };
341            let _ = app.emit("transfer-update", event);
342        }
343    }
344
345    fn is_multipart_upload_job(job: &TransferJob) -> bool {
346        if !matches!(job.transfer_type, TransferType::Upload) {
347            return false;
348        }
349
350        job.total_bytes >= MULTIPART_UPLOAD_THRESHOLD
351            || std::fs::metadata(&job.local_path)
352                .map(|metadata| metadata.len() >= MULTIPART_UPLOAD_THRESHOLD)
353                .unwrap_or(false)
354    }
355
356    async fn ensure_multipart_job_active(&self, id: &str) -> crate::error::Result<()> {
357        match self.get_job(id).await.map(|job| job.status) {
358            Some(TransferStatus::InProgress) => Ok(()),
359            Some(TransferStatus::Cancelled) | None => Err(crate::error::AppError::ConfigError(
360                "Multipart upload was cancelled".to_string(),
361            )),
362            Some(status) => Err(crate::error::AppError::ConfigError(format!(
363                "Multipart upload stopped because the transfer is no longer active ({status:?})"
364            ))),
365        }
366    }
367
368    async fn acquire_slot(&self) -> ActiveSlotGuard {
369        loop {
370            let notified = self.slot_notify.notified();
371            tokio::pin!(notified);
372            notified.as_mut().enable();
373            let max = self.max_concurrency.load(Ordering::Acquire).max(1);
374            let active = self.active_count.load(Ordering::Acquire);
375
376            if active < max {
377                if self
378                    .active_count
379                    .compare_exchange_weak(active, active + 1, Ordering::AcqRel, Ordering::Acquire)
380                    .is_ok()
381                {
382                    return ActiveSlotGuard {
383                        active_count: self.active_count.clone(),
384                        slot_notify: self.slot_notify.clone(),
385                    };
386                }
387                continue;
388            }
389
390            notified.await;
391        }
392    }
393
394    // Process the queue using a worker pool that respects max concurrency
395    pub async fn process_queue(
396        self: Arc<Self>,
397        s3_manager: Arc<RwLock<S3ClientManager>>,
398        profile_manager: Arc<RwLock<ProfileManager>>,
399    ) {
400        let manager = self.clone();
401
402        tokio::spawn(async move {
403            loop {
404                // 1. Get next job from queue
405                let next_id = {
406                    let mut queue = manager.queue.lock().await;
407                    if queue.is_empty() {
408                        break;
409                    }
410                    queue.pop_front().expect("queue checked above")
411                };
412
413                // 2. Wait for a slot in the concurrency limit
414                let slot_guard = manager.acquire_slot().await;
415
416                // 3. Spawn the task
417                let manager_inner = manager.clone();
418                let s3_inner = s3_manager.clone();
419                let profiles_inner = profile_manager.clone();
420                let id_inner = next_id.clone();
421
422                let handle = tokio::spawn(async move {
423                    let _slot_guard = slot_guard;
424                    let should_run = matches!(
425                        manager_inner.get_job(&id_inner).await.map(|job| job.status),
426                        Some(TransferStatus::Pending)
427                    );
428
429                    if !should_run {
430                        return;
431                    }
432
433                    // Update status to InProgress
434                    manager_inner
435                        .update_job_status(&id_inner, TransferStatus::InProgress)
436                        .await;
437
438                    // Run the job
439                    let job_opt = manager_inner.get_job(&id_inner).await;
440                    if let Some(job) = job_opt {
441                        if job.status != TransferStatus::InProgress {
442                            return;
443                        }
444                        let profile_result = {
445                            let profiles = profiles_inner.read().await;
446                            profiles.get_profile(&job.profile_id).await
447                        };
448
449                        let result = match profile_result {
450                            Ok(profile) => {
451                                let execution = manager_inner.execute_job(&job, s3_inner, &profile);
452                                tokio::pin!(execution);
453                                let mut timer = tokio::time::interval(Duration::from_millis(250));
454                                timer.set_missed_tick_behavior(
455                                    tokio::time::MissedTickBehavior::Skip,
456                                );
457                                let mut speed =
458                                    super::speed::SpeedWindow::new(std::time::Instant::now());
459                                loop {
460                                    tokio::select! {
461                                        result = &mut execution => break result,
462                                        _ = timer.tick() => {
463                                            let (position, attempt) = {
464                                                let progress = job.progress.lock().unwrap();
465                                                (progress.position, progress.attempt)
466                                            };
467                                            let rate = speed.sample(std::time::Instant::now(), position, attempt);
468                                            let mut jobs = manager_inner.jobs.write().await;
469                                            if let Some(current) = jobs.get_mut(&job.id) {
470                                                if current.status == TransferStatus::InProgress {
471                                                    current.processed_bytes = if current.total_bytes > 0 {
472                                                        position.min(current.total_bytes)
473                                                    } else {
474                                                        position
475                                                    };
476                                                    current.bytes_per_second = rate;
477                                                    manager_inner.emit_update(current).await;
478                                                }
479                                            }
480                                        }
481                                    }
482                                }
483                            }
484                            Err(err) => Err(err),
485                        };
486
487                        match result {
488                            Ok(_) => {
489                                // Double check if it was cancelled while we were working
490                                if let Some(current_job) = manager_inner.get_job(&id_inner).await {
491                                    if !matches!(current_job.status, TransferStatus::Cancelled) {
492                                        manager_inner
493                                            .update_job_status(&id_inner, TransferStatus::Completed)
494                                            .await;
495                                    }
496                                }
497                            }
498                            Err(e) => {
499                                let was_cancelled = matches!(
500                                    manager_inner.get_job(&id_inner).await.map(|job| job.status),
501                                    Some(TransferStatus::Cancelled) | None
502                                );
503                                if !was_cancelled {
504                                    log::error!("Transfer job {} failed: {}", id_inner, e);
505                                    manager_inner
506                                        .update_job_status(
507                                            &id_inner,
508                                            TransferStatus::Failed(e.to_string()),
509                                        )
510                                        .await;
511                                }
512                            }
513                        }
514                    }
515
516                    // Remove abort handle when done
517                    let mut handles = manager_inner.abort_handles.write().await;
518                    handles.remove(&id_inner);
519                });
520
521                // 4. Store the abort handle so we can cancel it later
522                let mut handles = manager.abort_handles.write().await;
523                handles.insert(next_id, handle.abort_handle());
524            }
525        });
526    }
527
528    async fn update_job_status(&self, id: &str, status: TransferStatus) {
529        {
530            let mut jobs = self.jobs.write().await;
531            if let Some(job) = jobs.get_mut(id) {
532                if matches!(
533                    job.status,
534                    TransferStatus::Cancelled
535                        | TransferStatus::Completed
536                        | TransferStatus::Failed(_)
537                ) && job.status != status
538                {
539                    return;
540                }
541                job.status = status.clone();
542                job.bytes_per_second = 0.0;
543                // If final status, set finished_at
544                match status {
545                    TransferStatus::Completed
546                    | TransferStatus::Failed(_)
547                    | TransferStatus::Cancelled => {
548                        job.finished_at = Some(chrono::Utc::now().timestamp_millis());
549                    }
550                    _ => {}
551                }
552            }
553        }
554        self.persist_or_log().await;
555        if let Some(job) = self.get_job(id).await {
556            self.emit_update(&job).await;
557        }
558    }
559
560    async fn update_job_total_size(&self, id: &str, size: u64) {
561        {
562            let mut jobs = self.jobs.write().await;
563            if let Some(job) = jobs.get_mut(id) {
564                job.total_bytes = size;
565            }
566        }
567        if let Some(job) = self.get_job(id).await {
568            self.emit_update(&job).await;
569        }
570    }
571
572    async fn update_job_progress(&self, id: &str, processed: u64) {
573        {
574            let mut jobs = self.jobs.write().await;
575            if let Some(job) = jobs.get_mut(id) {
576                if job.status != TransferStatus::InProgress {
577                    return;
578                }
579                job.processed_bytes = processed;
580                job.progress.lock().unwrap().position = processed;
581            }
582        }
583        if let Some(job) = self.get_job(id).await {
584            self.emit_update(&job).await;
585        }
586    }
587
588    async fn upload_file(
589        &self,
590        client: &Client,
591        job: &TransferJob,
592        content_type: &str,
593    ) -> crate::error::Result<()> {
594        let metadata = tokio::fs::metadata(&job.local_path)
595            .await
596            .map_err(|error| {
597                crate::error::AppError::IoError(format!(
598                    "Cannot read upload file '{}': {error}",
599                    job.local_path
600                ))
601            })?;
602        if !metadata.is_file() {
603            return Err(crate::error::AppError::IoError(format!(
604                "Upload source '{}' is not a regular file",
605                job.local_path
606            )));
607        }
608
609        let file_size = metadata.len();
610        if job.total_bytes != file_size {
611            self.update_job_total_size(&job.id, file_size).await;
612        }
613
614        if file_size < MULTIPART_UPLOAD_THRESHOLD {
615            let body = ByteStream::from_path(&job.local_path)
616                .await
617                .map_err(|error| crate::error::AppError::IoError(error.to_string()))?;
618            let body = super::speed::track(body, job.progress.clone(), 0);
619            let timeout = transfer_timeout_for(file_size);
620            let send = client
621                .put_object()
622                .bucket(&job.bucket)
623                .key(&job.key)
624                .content_type(content_type)
625                .if_none_match("*")
626                .body(body)
627                .send();
628            match tokio::time::timeout(timeout, send).await {
629                Ok(Ok(_)) => {}
630                Ok(Err(error)) => {
631                    return Err(crate::error::AppError::S3Error(format!(
632                        "Single-part upload failed for s3://{}/{}: {}",
633                        job.bucket,
634                        job.key,
635                        DisplayErrorContext(&error)
636                    )));
637                }
638                Err(_) => {
639                    return Err(crate::error::AppError::S3Error(format!(
640                        "Single-part upload timed out after {}s for s3://{}/{} with no response from S3 \
641                         (check the endpoint, region, network/proxy, and TLS configuration)",
642                        timeout.as_secs(),
643                        job.bucket,
644                        job.key
645                    )));
646                }
647            }
648            self.update_job_progress(&job.id, file_size).await;
649            return Ok(());
650        }
651
652        self.multipart_upload_file(client, job, content_type, file_size)
653            .await
654    }
655
656    async fn multipart_upload_file(
657        &self,
658        client: &Client,
659        job: &TransferJob,
660        content_type: &str,
661        file_size: u64,
662    ) -> crate::error::Result<()> {
663        self.ensure_multipart_job_active(&job.id).await?;
664        let plan = plan_multipart_upload(file_size)?;
665        let create_send = client
666            .create_multipart_upload()
667            .bucket(&job.bucket)
668            .key(&job.key)
669            .content_type(content_type)
670            .send();
671        let created = match tokio::time::timeout(METADATA_REQUEST_TIMEOUT, create_send).await {
672            Ok(Ok(output)) => output,
673            Ok(Err(error)) => {
674                return Err(crate::error::AppError::S3Error(format!(
675                    "Could not start multipart upload for s3://{}/{}: {}",
676                    job.bucket,
677                    job.key,
678                    DisplayErrorContext(&error)
679                )));
680            }
681            Err(_) => {
682                return Err(crate::error::AppError::S3Error(format!(
683                    "Starting multipart upload timed out after {}s for s3://{}/{} with no response from S3 \
684                     (check the endpoint, region, network/proxy, and TLS configuration)",
685                    METADATA_REQUEST_TIMEOUT.as_secs(),
686                    job.bucket,
687                    job.key
688                )));
689            }
690        };
691        let upload_id = created.upload_id().ok_or_else(|| {
692            crate::error::AppError::S3Error(format!(
693                "S3 did not return an upload ID for s3://{}/{}",
694                job.bucket, job.key
695            ))
696        })?;
697        let mut guard =
698            MultipartUploadGuard::new(client.clone(), &job.bucket, &job.key, upload_id.to_string());
699
700        let upload_result: crate::error::Result<()> = async {
701            self.ensure_multipart_job_active(&job.id).await?;
702            let mut completed_parts = Vec::with_capacity(plan.parts.len());
703            let mut uploaded_bytes = 0_u64;
704
705            for part in &plan.parts {
706                self.ensure_multipart_job_active(&job.id).await?;
707                let body = ByteStream::read_from()
708                    .path(&job.local_path)
709                    .offset(part.offset)
710                    .length(Length::Exact(part.length))
711                    .buffer_size(1024 * 1024)
712                    .build()
713                    .await
714                    .map_err(|error| {
715                        crate::error::AppError::IoError(format!(
716                            "Could not read part {} from '{}': {error}",
717                            part.number, job.local_path
718                        ))
719                    })?;
720
721                let body = super::speed::track(body, job.progress.clone(), part.offset);
722                let part_timeout = transfer_timeout_for(part.length);
723                let part_send = client
724                    .upload_part()
725                    .bucket(&job.bucket)
726                    .key(&job.key)
727                    .upload_id(guard.upload_id())
728                    .part_number(part.number)
729                    .content_length(part.length as i64)
730                    .body(body)
731                    .send();
732                let output = match tokio::time::timeout(part_timeout, part_send).await {
733                    Ok(Ok(output)) => output,
734                    Ok(Err(error)) => {
735                        return Err(crate::error::AppError::S3Error(format!(
736                            "Multipart upload failed at part {} of {} for s3://{}/{}: {}",
737                            part.number,
738                            plan.parts.len(),
739                            job.bucket,
740                            job.key,
741                            DisplayErrorContext(&error)
742                        )));
743                    }
744                    Err(_) => {
745                        return Err(crate::error::AppError::S3Error(format!(
746                            "Multipart upload timed out after {}s at part {} of {} for s3://{}/{} with no \
747                             response from S3 (check the endpoint, region, network/proxy, and TLS configuration)",
748                            part_timeout.as_secs(),
749                            part.number,
750                            plan.parts.len(),
751                            job.bucket,
752                            job.key
753                        )));
754                    }
755                };
756                self.ensure_multipart_job_active(&job.id).await?;
757                let e_tag = output.e_tag().ok_or_else(|| {
758                    crate::error::AppError::S3Error(format!(
759                        "S3 did not return an ETag for multipart part {} of s3://{}/{}",
760                        part.number, job.bucket, job.key
761                    ))
762                })?;
763
764                completed_parts.push(
765                    CompletedPart::builder()
766                        .part_number(part.number)
767                        .e_tag(e_tag)
768                        .build(),
769                );
770                uploaded_bytes += part.length;
771                self.update_job_progress(&job.id, uploaded_bytes).await;
772            }
773
774            self.ensure_multipart_job_active(&job.id).await?;
775            let completed_upload = CompletedMultipartUpload::builder()
776                .set_parts(Some(completed_parts))
777                .build();
778            let complete_send = client
779                .complete_multipart_upload()
780                .if_none_match("*")
781                .bucket(&job.bucket)
782                .key(&job.key)
783                .upload_id(guard.upload_id())
784                .multipart_upload(completed_upload)
785                .send();
786            match tokio::time::timeout(METADATA_REQUEST_TIMEOUT, complete_send).await {
787                Ok(Ok(_)) => {}
788                Ok(Err(error)) => {
789                    return Err(crate::error::AppError::S3Error(format!(
790                        "Could not complete multipart upload for s3://{}/{}: {}",
791                        job.bucket,
792                        job.key,
793                        DisplayErrorContext(&error)
794                    )));
795                }
796                Err(_) => {
797                    return Err(crate::error::AppError::S3Error(format!(
798                        "Completing multipart upload timed out after {}s for s3://{}/{} with no response from S3 \
799                         (check the endpoint, region, network/proxy, and TLS configuration)",
800                        METADATA_REQUEST_TIMEOUT.as_secs(),
801                        job.bucket,
802                        job.key
803                    )));
804                }
805            }
806
807            Ok(())
808        }
809        .await;
810
811        if let Err(error) = upload_result {
812            if let Err(abort_error) = guard.abort().await {
813                log::error!("{}", abort_error);
814            }
815            return Err(error);
816        }
817
818        guard.disarm();
819        Ok(())
820    }
821
822    async fn execute_job(
823        &self,
824        job: &TransferJob,
825        s3_manager: Arc<RwLock<S3ClientManager>>,
826        profile: &Profile,
827    ) -> crate::error::Result<()> {
828        let resolved_region = {
829            let s3 = s3_manager.read().await;
830            s3.get_bucket_region(profile, &job.bucket)
831        }
832        .or(job.bucket_region.clone());
833
834        let client = {
835            let mut s3 = s3_manager.write().await;
836            let c = if let Some(ref region) = resolved_region {
837                s3.get_client_for_region(profile, region).await?
838            } else {
839                s3.get_client(profile).await?
840            };
841            c.clone()
842        };
843
844        let detect_region = async {
845            let retry_client = {
846                let mut s3 = s3_manager.write().await;
847                s3.get_client(profile).await?.clone()
848            };
849
850            let new_region = crate::s3::get_bucket_region(&retry_client, &job.bucket)
851                .await
852                .ok();
853            if let Some(ref region) = new_region {
854                let mut s3 = s3_manager.write().await;
855                s3.set_bucket_region(profile, &job.bucket, region.clone());
856            }
857            Ok::<Option<String>, crate::error::AppError>(new_region)
858        };
859
860        match job.transfer_type {
861            TransferType::Upload => {
862                let content_type = crate::s3::infer_content_type(&job.key);
863                if let Err(first_error) = self.upload_file(&client, job, &content_type).await {
864                    if !matches!(
865                        self.get_job(&job.id).await.map(|current| current.status),
866                        Some(TransferStatus::InProgress)
867                    ) {
868                        return Err(first_error);
869                    }
870                    log::warn!(
871                        "Upload failed using configured region {:?}; checking the bucket region once before returning the error: {}",
872                        resolved_region,
873                        first_error
874                    );
875
876                    let detected_region = detect_region.await?;
877                    let retry_region = detected_region
878                        .filter(|region| resolved_region.as_deref() != Some(region.as_str()));
879                    if let Some(new_region) = retry_region {
880                        let retry_client = {
881                            let mut s3 = s3_manager.write().await;
882                            s3.get_client_for_region(profile, &new_region)
883                                .await?
884                                .clone()
885                        };
886                        self.update_job_progress(&job.id, 0).await;
887                        self.upload_file(&retry_client, job, &content_type)
888                            .await
889                            .map_err(|error| {
890                                crate::error::AppError::S3Error(format!(
891                                    "Upload retry in detected bucket region '{new_region}' failed: {error}"
892                                ))
893                            })?;
894                    } else {
895                        return Err(first_error);
896                    }
897                }
898
899                {
900                    let mut s3 = s3_manager.write().await;
901                    s3.remove_bucket_cache(&profile.id, &job.bucket);
902                }
903            }
904            TransferType::Download => {
905                let result = tokio::time::timeout(
906                    METADATA_REQUEST_TIMEOUT,
907                    client.get_object().bucket(&job.bucket).key(&job.key).send(),
908                )
909                .await;
910
911                let mut output = match result {
912                    Ok(Ok(output)) => output,
913                    Err(_) => {
914                        return Err(crate::error::AppError::S3Error(format!(
915                            "Download timed out after {}s for s3://{}/{} with no response from S3 \
916                             (check the endpoint, region, network/proxy, and TLS configuration)",
917                            METADATA_REQUEST_TIMEOUT.as_secs(),
918                            job.bucket,
919                            job.key
920                        )));
921                    }
922                    Ok(Err(err)) => {
923                        log::warn!(
924                            "download transfer failed, attempting region discovery: {}",
925                            DisplayErrorContext(&err)
926                        );
927
928                        if let Some(new_region) = detect_region.await? {
929                            let retry_client = {
930                                let mut s3 = s3_manager.write().await;
931                                s3.get_client_for_region(profile, &new_region)
932                                    .await?
933                                    .clone()
934                            };
935
936                            tokio::time::timeout(
937                                METADATA_REQUEST_TIMEOUT,
938                                retry_client
939                                    .get_object()
940                                    .bucket(&job.bucket)
941                                    .key(&job.key)
942                                    .send(),
943                            )
944                            .await
945                            .map_err(|_| {
946                                crate::error::AppError::S3Error(format!(
947                                    "Retry download timed out after {}s for s3://{}/{} with no response \
948                                     from S3 (check the endpoint, region, network/proxy, and TLS configuration)",
949                                    METADATA_REQUEST_TIMEOUT.as_secs(),
950                                    job.bucket,
951                                    job.key
952                                ))
953                            })?
954                            .map_err(|e| {
955                                crate::error::AppError::S3Error(format!(
956                                    "Retry download failed: {}",
957                                    DisplayErrorContext(&e)
958                                ))
959                                })?
960                        } else {
961                            return Err(crate::error::AppError::S3Error(format!(
962                                "{}",
963                                DisplayErrorContext(&err)
964                            )));
965                        }
966                    }
967                };
968
969                let destination = job.download_destination.as_ref().ok_or_else(|| {
970                    crate::error::AppError::IoError(
971                        "Missing download destination. Queue the download again.".to_string(),
972                    )
973                })?;
974                let mut download = destination.begin()?;
975
976                let mut downloaded: u64 = 0;
977                let mut last_update = std::time::Instant::now();
978
979                while let Some(bytes) =
980                    tokio::time::timeout(METADATA_REQUEST_TIMEOUT, output.body.try_next())
981                        .await
982                        .map_err(|_| {
983                            crate::error::AppError::S3Error(
984                                "Download stalled for 60 seconds. Check the connection and retry."
985                                    .into(),
986                            )
987                        })?
988                        .map_err(|e| {
989                            crate::error::AppError::S3Error(format!("{}", DisplayErrorContext(&e)))
990                        })?
991                {
992                    download
993                        .write_all(&bytes)
994                        .await
995                        .map_err(|e| crate::error::AppError::IoError(e.to_string()))?;
996
997                    downloaded += bytes.len() as u64;
998
999                    if last_update.elapsed() >= std::time::Duration::from_millis(100) {
1000                        self.update_job_progress(&job.id, downloaded).await;
1001                        last_update = std::time::Instant::now();
1002                    }
1003                }
1004
1005                self.update_job_progress(&job.id, downloaded).await;
1006                if job.total_bytes == 0 {
1007                    self.update_job_total_size(&job.id, downloaded).await;
1008                }
1009                download.finish_writing().await?;
1010                let mut jobs = self.jobs.write().await;
1011                let current = jobs.get_mut(&job.id).ok_or_else(|| {
1012                    crate::error::AppError::IoError("Download was removed".to_string())
1013                })?;
1014                if current.status != TransferStatus::InProgress {
1015                    return Err(crate::error::AppError::IoError(
1016                        "Download was cancelled".to_string(),
1017                    ));
1018                }
1019                download.commit()?;
1020                current.status = TransferStatus::Completed;
1021                current.finished_at = Some(chrono::Utc::now().timestamp_millis());
1022                drop(jobs);
1023                self.persist_or_log().await;
1024            }
1025        }
1026
1027        Ok(())
1028    }
1029}
1030
1031#[cfg(test)]
1032mod tests {
1033    use super::{Client, RwLock, S3ClientManager, TransferManager};
1034
1035    #[tokio::test]
1036    async fn interrupted_jobs_are_recovered_without_restarting_writes() {
1037        let directory = tempfile::tempdir().unwrap();
1038        let path = directory.path().join("transfers.json");
1039        let manager = TransferManager::new();
1040        manager.configure_journal(path.clone()).await.unwrap();
1041        let job = TransferJob::new(
1042            TransferType::Upload,
1043            "profile".into(),
1044            "bucket".into(),
1045            None,
1046            "key".into(),
1047            "/unused".into(),
1048            10,
1049        );
1050        let id = job.id.clone();
1051        manager.add_job(job).await;
1052        let restarted = TransferManager::new();
1053        restarted.configure_journal(path).await.unwrap();
1054        assert!(matches!(
1055            restarted.get_job(&id).await.unwrap().status,
1056            TransferStatus::Failed(_)
1057        ));
1058        assert!(restarted.queue.lock().await.is_empty());
1059    }
1060
1061    #[tokio::test]
1062    async fn corrupt_recovery_journal_is_preserved_without_blocking_startup() {
1063        let directory = tempfile::tempdir().unwrap();
1064        let path = directory.path().join("transfers.json");
1065        std::fs::write(&path, "broken").unwrap();
1066        let manager = TransferManager::new();
1067        manager.configure_journal(path).await.unwrap();
1068        assert!(manager.list_jobs().await.is_empty());
1069        let backup = std::fs::read_dir(directory.path())
1070            .unwrap()
1071            .map(|entry| entry.unwrap().path())
1072            .find(|path| {
1073                path.file_name()
1074                    .unwrap()
1075                    .to_string_lossy()
1076                    .starts_with("transfers.invalid-")
1077            })
1078            .unwrap();
1079        assert_eq!(std::fs::read_to_string(backup).unwrap(), "broken");
1080    }
1081    use crate::credentials::{CredentialType, Profile};
1082    use crate::transfer::{TransferJob, TransferStatus, TransferType};
1083    use std::path::PathBuf;
1084    use std::sync::Arc;
1085    use std::time::Duration;
1086
1087    #[test]
1088    fn upload_timeouts_cover_small_and_large_payloads_without_overflow() {
1089        assert_eq!(super::transfer_timeout_for(0), Duration::from_secs(60));
1090        assert_eq!(
1091            super::transfer_timeout_for(10 * 1024 * 1024),
1092            Duration::from_secs(103)
1093        );
1094        assert_eq!(
1095            super::transfer_timeout_for(u64::MAX),
1096            Duration::from_secs(1800)
1097        );
1098    }
1099
1100    #[tokio::test]
1101    async fn stalled_download_releases_its_partial_file() {
1102        let (endpoint, received, server) = crate::commands::test_s3::stalled_endpoint(
1103            "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ndo",
1104        )
1105        .await;
1106        let profile = Profile::new(
1107            "test".into(),
1108            CredentialType::CustomEndpoint {
1109                endpoint_url: endpoint,
1110                access_key_id: "TEST".into(),
1111                secret_access_key: "test-secret".into(),
1112            },
1113            Some("us-east-1".into()),
1114        );
1115        let directory = tempfile::tempdir().unwrap();
1116        let path = directory.path().join("result.txt");
1117        let mut job = test_job(TransferStatus::InProgress);
1118        job.profile_id = profile.id.clone();
1119        job.local_path = path.to_string_lossy().into_owned();
1120        job.download_destination = Some(Arc::new(
1121            crate::transfer::download::DownloadDestination::from_path(&path, false).unwrap(),
1122        ));
1123        let manager = TransferManager::new();
1124        manager.add_job(job.clone()).await;
1125        let task = tokio::spawn(async move {
1126            manager
1127                .execute_job(
1128                    &job,
1129                    Arc::new(RwLock::new(S3ClientManager::new())),
1130                    &profile,
1131                )
1132                .await
1133        });
1134        received.await.unwrap();
1135        tokio::time::pause();
1136        let result = task.await.unwrap();
1137        tokio::time::resume();
1138        server.abort();
1139        assert!(result.is_err());
1140        assert!(!path.exists());
1141        assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0);
1142    }
1143
1144    #[tokio::test]
1145    async fn failed_upload_retains_the_provider_error() {
1146        let (client, server) =
1147            crate::commands::test_s3::scripted_client(vec![crate::commands::test_s3::response(
1148                403,
1149                "",
1150                "<Error><Code>AccessDenied</Code><Message>Uploads are disabled</Message></Error>",
1151            )])
1152            .await;
1153        let client = Client::from_conf(
1154            client
1155                .config()
1156                .to_builder()
1157                .request_checksum_calculation(
1158                    aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired,
1159                )
1160                .build(),
1161        );
1162        let directory = tempfile::tempdir().unwrap();
1163        let path = directory.path().join("input.txt");
1164        std::fs::write(&path, b"test").unwrap();
1165        let mut job = test_job(TransferStatus::InProgress);
1166        job.local_path = path.to_string_lossy().into_owned();
1167        let error = TransferManager::new()
1168            .upload_file(&client, &job, "text/plain")
1169            .await
1170            .unwrap_err();
1171        assert!(error.to_string().contains("AccessDenied"));
1172        assert!(error.to_string().contains("Uploads are disabled"));
1173        assert_eq!(server.await.unwrap().len(), 1);
1174    }
1175
1176    #[tokio::test]
1177    async fn retried_download_retains_its_destination_and_profile() {
1178        let directory = tempfile::tempdir().unwrap();
1179        let path = directory.path().join("result.txt");
1180        let destination = Arc::new(
1181            crate::transfer::download::DownloadDestination::from_path(&path, false).unwrap(),
1182        );
1183        let mut job = TransferJob::new(
1184            TransferType::Download,
1185            "profile-a".into(),
1186            "bucket".into(),
1187            Some("region".into()),
1188            "folder/result.txt".into(),
1189            path.clone(),
1190            4,
1191        )
1192        .with_group("group".into(), "folder".into());
1193        job.download_destination = Some(destination);
1194        job.status = TransferStatus::Failed("Connection lost".into());
1195        job.processed_bytes = 2;
1196        let manager = TransferManager::new();
1197        manager.add_job(job.clone()).await;
1198        let retry_id = manager.retry_job(&job.id).await.unwrap();
1199        let retry = manager.get_job(&retry_id).await.unwrap();
1200        assert_ne!(retry_id, job.id);
1201        assert_eq!(retry.profile_id, "profile-a");
1202        assert_eq!(retry.parent_group_id, job.parent_group_id);
1203        assert_eq!(retry.group_name, job.group_name);
1204        assert_eq!(retry.status, TransferStatus::Pending);
1205        assert_eq!(retry.processed_bytes, 0);
1206        assert!(matches!(
1207            manager.get_job(&job.id).await.unwrap().status,
1208            TransferStatus::Failed(_)
1209        ));
1210        let mut pending = retry
1211            .download_destination
1212            .expect("A retry must retain the selected download directory")
1213            .begin()
1214            .unwrap();
1215        pending.write_all(b"done").await.unwrap();
1216        pending.finish_writing().await.unwrap();
1217        pending.commit().unwrap();
1218        assert_eq!(std::fs::read(path).unwrap(), b"done");
1219    }
1220
1221    fn test_job(status: TransferStatus) -> TransferJob {
1222        let mut job = TransferJob::new(
1223            TransferType::Download,
1224            "profile-a".into(),
1225            "bucket".into(),
1226            None,
1227            "file.txt".into(),
1228            PathBuf::from("file.txt"),
1229            10,
1230        );
1231        job.status = status;
1232        job
1233    }
1234
1235    #[tokio::test]
1236    async fn terminal_jobs_clear_speed_and_ignore_late_progress() {
1237        for status in [
1238            TransferStatus::Completed,
1239            TransferStatus::Failed("offline".into()),
1240            TransferStatus::Cancelled,
1241        ] {
1242            let manager = TransferManager::new();
1243            let mut job = test_job(TransferStatus::InProgress);
1244            job.bytes_per_second = 1024.0;
1245            manager.add_job(job.clone()).await;
1246            if status == TransferStatus::Cancelled {
1247                manager.cancel_job(&job.id).await;
1248            } else {
1249                manager.update_job_status(&job.id, status.clone()).await;
1250            }
1251            manager.update_job_progress(&job.id, 5).await;
1252            let saved = manager.get_job(&job.id).await.unwrap();
1253            assert_eq!(saved.status, status);
1254            assert_eq!(saved.bytes_per_second, 0.0);
1255            assert_eq!(saved.processed_bytes, 0);
1256        }
1257    }
1258
1259    #[tokio::test]
1260    async fn cancelling_pending_work_keeps_history_and_removes_it_from_the_queue() {
1261        let manager = TransferManager::new();
1262        let job = test_job(TransferStatus::Pending);
1263        manager.add_job(job.clone()).await;
1264        assert!(manager.cancel_job(&job.id).await);
1265        assert_eq!(
1266            manager.get_job(&job.id).await.unwrap().status,
1267            TransferStatus::Cancelled
1268        );
1269        assert!(!manager.queue.lock().await.contains(&job.id));
1270        assert!(!manager.cancel_job(&job.id).await);
1271        assert!(!manager.cancel_job("missing").await);
1272    }
1273
1274    #[tokio::test]
1275    async fn cancelling_active_downloads_aborts_the_task_and_releases_capacity() {
1276        let manager = TransferManager::new();
1277        let job = test_job(TransferStatus::InProgress);
1278        manager.add_job(job.clone()).await;
1279        let slot = manager.acquire_slot().await;
1280        let task = tokio::spawn(async move {
1281            let _slot = slot;
1282            std::future::pending::<()>().await;
1283        });
1284        manager
1285            .abort_handles
1286            .write()
1287            .await
1288            .insert(job.id.clone(), task.abort_handle());
1289        assert!(manager.cancel_job(&job.id).await);
1290        assert!(tokio::time::timeout(Duration::from_secs(2), task)
1291            .await
1292            .unwrap()
1293            .unwrap_err()
1294            .is_cancelled());
1295        assert_eq!(
1296            manager
1297                .active_count
1298                .load(std::sync::atomic::Ordering::Acquire),
1299            0
1300        );
1301    }
1302
1303    #[tokio::test]
1304    async fn clearing_history_retains_pending_and_active_transfers() {
1305        let manager = TransferManager::new();
1306        let pending = test_job(TransferStatus::Pending);
1307        let active = test_job(TransferStatus::InProgress);
1308        for job in [
1309            pending.clone(),
1310            active.clone(),
1311            test_job(TransferStatus::Completed),
1312            test_job(TransferStatus::Failed("offline".into())),
1313            test_job(TransferStatus::Cancelled),
1314        ] {
1315            manager.add_job(job).await;
1316        }
1317        assert_eq!(manager.clear_completed().await, 3);
1318        let jobs = manager.list_jobs().await;
1319        assert_eq!(jobs.len(), 2);
1320        assert!(jobs.iter().any(|job| job.id == pending.id));
1321        assert!(jobs.iter().any(|job| job.id == active.id));
1322        assert_eq!(manager.clear_completed().await, 0);
1323    }
1324
1325    #[tokio::test]
1326    async fn retry_refuses_active_and_completed_work() {
1327        let manager = TransferManager::new();
1328        for status in [
1329            TransferStatus::Pending,
1330            TransferStatus::InProgress,
1331            TransferStatus::Completed,
1332        ] {
1333            let job = test_job(status);
1334            manager.add_job(job.clone()).await;
1335            assert!(manager.retry_job(&job.id).await.is_none());
1336        }
1337        assert!(manager.retry_job("missing").await.is_none());
1338        assert_eq!(manager.list_jobs().await.len(), 3);
1339    }
1340
1341    #[tokio::test]
1342    async fn removing_pending_work_removes_both_its_queue_entry_and_history() {
1343        let manager = TransferManager::new();
1344        let job = test_job(TransferStatus::Pending);
1345        manager.add_job(job.clone()).await;
1346        assert!(manager.remove_job(&job.id).await);
1347        assert!(manager.get_job(&job.id).await.is_none());
1348        assert!(!manager.queue.lock().await.contains(&job.id));
1349        assert!(!manager.remove_job(&job.id).await);
1350    }
1351
1352    #[tokio::test]
1353    async fn transfer_slots_follow_concurrency_changes_without_stalling() {
1354        let manager = TransferManager::new();
1355        manager.set_max_concurrency(1);
1356        let first = manager.acquire_slot().await;
1357        let second = manager.acquire_slot();
1358        tokio::pin!(second);
1359        assert!(futures::poll!(second.as_mut()).is_pending());
1360        manager.set_max_concurrency(2);
1361        let second = tokio::time::timeout(Duration::from_secs(1), second)
1362            .await
1363            .unwrap();
1364        manager.set_max_concurrency(1);
1365        let third = manager.acquire_slot();
1366        tokio::pin!(third);
1367        assert!(futures::poll!(third.as_mut()).is_pending());
1368        drop(second);
1369        assert!(futures::poll!(third.as_mut()).is_pending());
1370        drop(first);
1371        tokio::time::timeout(Duration::from_secs(1), third)
1372            .await
1373            .unwrap();
1374    }
1375
1376    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1377    async fn released_slots_allow_all_waiting_transfers_to_finish() {
1378        use std::sync::atomic::{AtomicUsize, Ordering};
1379        let manager = Arc::new(TransferManager::new());
1380        manager.set_max_concurrency(3);
1381        let peak = Arc::new(AtomicUsize::new(0));
1382        let tasks = (0..200)
1383            .map(|_| {
1384                let manager = manager.clone();
1385                let peak = peak.clone();
1386                tokio::spawn(async move {
1387                    let _slot = manager.acquire_slot().await;
1388                    peak.fetch_max(
1389                        manager.active_count.load(Ordering::Acquire),
1390                        Ordering::AcqRel,
1391                    );
1392                    tokio::task::yield_now().await;
1393                })
1394            })
1395            .collect::<Vec<_>>();
1396        let results =
1397            tokio::time::timeout(Duration::from_secs(5), futures::future::join_all(tasks))
1398                .await
1399                .unwrap();
1400        assert!(results.into_iter().all(|result| result.is_ok()));
1401        assert!(peak.load(Ordering::Acquire) <= 3);
1402        assert_eq!(manager.active_count.load(Ordering::Acquire), 0);
1403    }
1404
1405    /// Run with BROWS3_S3_TEST_ENDPOINT=http://127.0.0.1:<port> against MinIO.
1406    /// The sparse 129 MiB source crosses the production multipart threshold
1407    /// and the default 128 MiB part size.
1408    #[tokio::test]
1409    #[ignore = "requires an S3-compatible integration-test endpoint"]
1410    async fn multipart_upload_round_trips_against_s3_endpoint() {
1411        let endpoint = std::env::var("BROWS3_S3_TEST_ENDPOINT")
1412            .expect("BROWS3_S3_TEST_ENDPOINT must identify the test S3 endpoint");
1413        let access_key =
1414            std::env::var("BROWS3_S3_TEST_ACCESS_KEY").unwrap_or_else(|_| "minioadmin".into());
1415        let secret_key =
1416            std::env::var("BROWS3_S3_TEST_SECRET_KEY").unwrap_or_else(|_| "minioadmin".into());
1417        let profile = Profile::new(
1418            "multipart-integration".to_string(),
1419            CredentialType::CustomEndpoint {
1420                endpoint_url: endpoint,
1421                access_key_id: access_key,
1422                secret_access_key: secret_key,
1423            },
1424            Some("us-east-1".to_string()),
1425        );
1426        let sdk_config = crate::s3::client::load_sdk_config(&profile, None).await;
1427        let client = crate::s3::client::client_from_sdk_config(&sdk_config, &profile);
1428        let bucket = format!("brows3-multipart-test-{}", uuid::Uuid::new_v4().simple());
1429        client
1430            .create_bucket()
1431            .bucket(&bucket)
1432            .send()
1433            .await
1434            .expect("test bucket should be created");
1435
1436        let path: PathBuf = std::env::temp_dir().join(format!(
1437            "brows3-multipart-{}.bin",
1438            uuid::Uuid::new_v4().simple()
1439        ));
1440        let file = tokio::fs::File::create(&path)
1441            .await
1442            .expect("sparse test file should be created");
1443        let size = 129 * 1024 * 1024;
1444        file.set_len(size)
1445            .await
1446            .expect("sparse test file should be sized");
1447        drop(file);
1448
1449        let manager = Arc::new(TransferManager::new());
1450        let job = TransferJob::new(
1451            TransferType::Upload,
1452            profile.id.clone(),
1453            bucket.clone(),
1454            profile.region.clone(),
1455            "large/test.bin".to_string(),
1456            path.clone(),
1457            size,
1458        );
1459        manager.add_job(job.clone()).await;
1460        manager
1461            .update_job_status(&job.id, TransferStatus::InProgress)
1462            .await;
1463        manager
1464            .upload_file(&client, &job, "application/octet-stream")
1465            .await
1466            .expect("multipart upload should complete");
1467
1468        let head = client
1469            .head_object()
1470            .bucket(&bucket)
1471            .key(&job.key)
1472            .send()
1473            .await
1474            .expect("uploaded object should be readable");
1475        assert_eq!(head.content_length(), Some(size as i64));
1476        assert_eq!(head.content_type(), Some("application/octet-stream"));
1477        assert!(head.e_tag().is_some_and(|e_tag| e_tag.contains('-')));
1478
1479        let incomplete = client
1480            .list_multipart_uploads()
1481            .bucket(&bucket)
1482            .send()
1483            .await
1484            .expect("multipart uploads should be listable");
1485        assert!(incomplete.uploads().is_empty());
1486
1487        let cancel_path: PathBuf = std::env::temp_dir().join(format!(
1488            "brows3-multipart-cancel-{}.bin",
1489            uuid::Uuid::new_v4().simple()
1490        ));
1491        let cancel_file = tokio::fs::File::create(&cancel_path)
1492            .await
1493            .expect("sparse cancellation file should be created");
1494        let cancel_size = 2 * 1024 * 1024 * 1024;
1495        cancel_file
1496            .set_len(cancel_size)
1497            .await
1498            .expect("sparse cancellation file should be sized");
1499        drop(cancel_file);
1500        let cancel_job = TransferJob::new(
1501            TransferType::Upload,
1502            profile.id.clone(),
1503            bucket.clone(),
1504            profile.region.clone(),
1505            "large/cancelled.bin".to_string(),
1506            cancel_path.clone(),
1507            cancel_size,
1508        );
1509        manager.add_job(cancel_job.clone()).await;
1510        manager
1511            .update_job_status(&cancel_job.id, TransferStatus::InProgress)
1512            .await;
1513
1514        let upload_task = {
1515            let task_manager = manager.clone();
1516            let task_client = client.clone();
1517            let task_job = cancel_job.clone();
1518            tokio::spawn(async move {
1519                task_manager
1520                    .multipart_upload_file(
1521                        &task_client,
1522                        &task_job,
1523                        "application/octet-stream",
1524                        cancel_size,
1525                    )
1526                    .await
1527            })
1528        };
1529
1530        let mut upload_started = false;
1531        for _ in 0..100 {
1532            let active = client
1533                .list_multipart_uploads()
1534                .bucket(&bucket)
1535                .send()
1536                .await
1537                .expect("multipart uploads should be listable");
1538            if !active.uploads().is_empty() {
1539                upload_started = true;
1540                break;
1541            }
1542            tokio::time::sleep(Duration::from_millis(20)).await;
1543        }
1544        assert!(upload_started, "test multipart upload should start");
1545        assert!(manager.cancel_job(&cancel_job.id).await);
1546        let cancelled = tokio::time::timeout(Duration::from_secs(30), upload_task)
1547            .await
1548            .expect("cancelled multipart upload should finish cleanup")
1549            .expect("multipart task should not panic");
1550        assert!(cancelled.is_err());
1551        let incomplete = client
1552            .list_multipart_uploads()
1553            .bucket(&bucket)
1554            .send()
1555            .await
1556            .expect("multipart uploads should be listable after cancellation");
1557        assert!(incomplete.uploads().is_empty());
1558        assert!(client
1559            .head_object()
1560            .bucket(&bucket)
1561            .key(&cancel_job.key)
1562            .send()
1563            .await
1564            .is_err());
1565
1566        client
1567            .delete_object()
1568            .bucket(&bucket)
1569            .key(&job.key)
1570            .send()
1571            .await
1572            .expect("test object should be deleted");
1573        client
1574            .delete_bucket()
1575            .bucket(&bucket)
1576            .send()
1577            .await
1578            .expect("test bucket should be deleted");
1579        tokio::fs::remove_file(path)
1580            .await
1581            .expect("sparse test file should be deleted");
1582        tokio::fs::remove_file(cancel_path)
1583            .await
1584            .expect("sparse cancellation file should be deleted");
1585    }
1586}