use serde::{Deserialize, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use tokio::sync::{Mutex, Semaphore};
pub type PauseSignal = Arc<AtomicBool>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct JobId(u64);
impl JobId {
fn new(id: u64) -> Self {
JobId(id)
}
}
impl From<u64> for JobId {
fn from(id: u64) -> Self {
JobId(id)
}
}
impl std::fmt::Display for JobId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Job-{}", self.0)
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum TransferStatus {
Queued,
Active { progress: f64 },
Paused { progress: f64 },
Completed,
Failed { error: String },
Cancelled,
}
#[derive(Debug, Clone)]
pub struct TransferJob {
pub id: JobId,
pub status: TransferStatus,
pub priority: i32,
}
impl TransferJob {
fn new(id: JobId) -> Self {
TransferJob {
id,
status: TransferStatus::Queued,
priority: 0,
}
}
}
#[derive(Debug)]
struct TransferQueue {
jobs: VecDeque<TransferJob>,
}
impl TransferQueue {
fn new() -> Self {
TransferQueue {
jobs: VecDeque::new(),
}
}
fn enqueue(&mut self, job: TransferJob) {
let pos = self
.jobs
.iter()
.position(|j| j.priority < job.priority)
.unwrap_or(self.jobs.len());
self.jobs.insert(pos, job);
}
fn dequeue(&mut self) -> Option<TransferJob> {
self.jobs.pop_front()
}
fn get(&self, job_id: JobId) -> Option<&TransferJob> {
self.jobs.iter().find(|j| j.id == job_id)
}
fn remove(&mut self, job_id: JobId) -> Option<TransferJob> {
if let Some(pos) = self.jobs.iter().position(|j| j.id == job_id) {
self.jobs.remove(pos)
} else {
None
}
}
fn prioritize(&mut self, job_id: JobId) {
if let Some(pos) = self.jobs.iter().position(|j| j.id == job_id) {
if let Some(job) = self.jobs.remove(pos) {
self.jobs.push_front(job);
}
}
}
}
pub struct TransferManager {
next_job_id: AtomicU64,
pending: Arc<Mutex<TransferQueue>>,
active: Arc<Mutex<Vec<TransferJob>>>,
paused: Arc<Mutex<Vec<TransferJob>>>,
history: Arc<Mutex<Vec<TransferJob>>>,
semaphore: Arc<Semaphore>,
pause_signals: Arc<Mutex<HashMap<JobId, PauseSignal>>>,
}
impl TransferManager {
pub fn new(concurrency: usize) -> Self {
TransferManager {
next_job_id: AtomicU64::new(1),
pending: Arc::new(Mutex::new(TransferQueue::new())),
active: Arc::new(Mutex::new(Vec::new())),
paused: Arc::new(Mutex::new(Vec::new())),
history: Arc::new(Mutex::new(Vec::new())),
semaphore: Arc::new(Semaphore::new(concurrency)),
pause_signals: Arc::new(Mutex::new(HashMap::new())),
}
}
fn generate_job_id(&self) -> JobId {
JobId::new(self.next_job_id.fetch_add(1, Ordering::SeqCst))
}
pub async fn enqueue_upload(&self, _local_path: String, _s3_path: String, _size: Option<u64>) -> JobId {
let job_id = self.generate_job_id();
let job = TransferJob::new(job_id);
self.pending.lock().await.enqueue(job);
job_id
}
pub async fn enqueue_download(&self, _s3_path: String, _local_path: String, _size: Option<u64>) -> JobId {
let job_id = self.generate_job_id();
let job = TransferJob::new(job_id);
self.pending.lock().await.enqueue(job);
job_id
}
pub async fn pause(&self, job_id: JobId) -> Result<(), String> {
{
let mut active = self.active.lock().await;
if let Some(pos) = active.iter().position(|j| j.id == job_id) {
let mut job = active.remove(pos);
if let TransferStatus::Active { progress } = job.status {
job.status = TransferStatus::Paused { progress };
}
self.paused.lock().await.push(job);
if let Some(signal) = self.pause_signals.lock().await.remove(&job_id) {
signal.store(true, Ordering::SeqCst);
}
drop(active);
self.release_slot();
return Ok(());
}
}
{
let mut pending = self.pending.lock().await;
if let Some(job) = pending.remove(job_id) {
let progress = match job.status {
TransferStatus::Active { progress } => progress,
TransferStatus::Queued => 0.0,
_ => 0.0,
};
let mut paused_job = job;
paused_job.status = TransferStatus::Paused { progress };
self.paused.lock().await.push(paused_job);
return Ok(());
}
}
Err(format!("Job {} is not active or pending", job_id))
}
pub async fn resume(&self, job_id: JobId) -> Result<(), String> {
let mut paused = self.paused.lock().await;
if let Some(pos) = paused.iter().position(|j| j.id == job_id) {
let mut job = paused.remove(pos);
if let TransferStatus::Paused { progress } = job.status {
job.status = TransferStatus::Active { progress };
} else {
job.status = TransferStatus::Queued;
}
let mut pending = self.pending.lock().await;
pending.prioritize(job_id);
if pending.get(job_id).is_none() {
job.priority = i32::MAX; pending.enqueue(job);
}
Ok(())
} else {
Err(format!("Job {} is not paused", job_id))
}
}
pub async fn cancel(&self, job_id: JobId) -> Result<(), String> {
{
let mut pending = self.pending.lock().await;
if let Some(mut job) = pending.remove(job_id) {
job.status = TransferStatus::Cancelled;
self.history.lock().await.push(job);
return Ok(());
}
}
{
let mut paused = self.paused.lock().await;
if let Some(pos) = paused.iter().position(|j| j.id == job_id) {
let mut job = paused.remove(pos);
job.status = TransferStatus::Cancelled;
self.history.lock().await.push(job);
return Ok(());
}
}
{
let mut active = self.active.lock().await;
if let Some(pos) = active.iter().position(|j| j.id == job_id) {
let mut job = active.remove(pos);
job.status = TransferStatus::Cancelled;
self.history.lock().await.push(job);
if let Some(signal) = self.pause_signals.lock().await.remove(&job_id) {
signal.store(true, Ordering::SeqCst);
}
drop(active); self.release_slot();
return Ok(());
}
}
Err(format!("Job {} not found", job_id))
}
pub async fn mark_completed(&self, job_id: JobId) {
let mut active = self.active.lock().await;
if let Some(pos) = active.iter().position(|j| j.id == job_id) {
let mut job = active.remove(pos);
job.status = TransferStatus::Completed;
self.history.lock().await.push(job);
self.pause_signals.lock().await.remove(&job_id);
drop(active); self.release_slot();
}
}
pub async fn mark_failed(&self, job_id: JobId, error: String) {
let mut active = self.active.lock().await;
if let Some(pos) = active.iter().position(|j| j.id == job_id) {
let mut job = active.remove(pos);
job.status = TransferStatus::Failed { error };
self.history.lock().await.push(job);
self.pause_signals.lock().await.remove(&job_id);
drop(active); self.release_slot();
}
}
pub async fn try_get_next(&self) -> Option<(TransferJob, PauseSignal)> {
let available = self.semaphore.available_permits();
if available == 0 {
return None;
}
let job = {
let mut pending = self.pending.lock().await;
pending.dequeue()
};
if let Some(mut job) = job {
let _permit = self.semaphore.try_acquire().ok()?;
std::mem::forget(_permit);
job.status = TransferStatus::Active { progress: 0.0 };
let job_id = job.id;
self.active.lock().await.push(job.clone());
let pause_signal = Arc::new(AtomicBool::new(false));
self.pause_signals
.lock()
.await
.insert(job_id, pause_signal.clone());
Some((job, pause_signal))
} else {
None
}
}
pub fn release_slot(&self) {
self.semaphore.add_permits(1);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_job_id_generation() {
let manager = TransferManager::new(4);
let id1 = manager.enqueue_upload("local1".into(), "s3/path1".into(), Some(100)).await;
let id2 = manager.enqueue_upload("local2".into(), "s3/path2".into(), Some(200)).await;
assert_ne!(id1, id2);
}
#[tokio::test]
async fn test_transfer_queue_ordering() {
let manager = TransferManager::new(4);
let id1 = manager.enqueue_upload("file1".into(), "s3/file1".into(), None).await;
let id2 = manager.enqueue_upload("file2".into(), "s3/file2".into(), None).await;
let id3 = manager.enqueue_upload("file3".into(), "s3/file3".into(), None).await;
let (job1, _) = manager.try_get_next().await.unwrap();
assert_eq!(job1.id, id1);
let (job2, _) = manager.try_get_next().await.unwrap();
assert_eq!(job2.id, id2);
let (job3, _) = manager.try_get_next().await.unwrap();
assert_eq!(job3.id, id3);
}
#[tokio::test]
async fn test_transfer_pause_resume() {
let manager = TransferManager::new(4);
let job_id = manager.enqueue_upload("file".into(), "s3/file".into(), None).await;
let (job, _) = manager.try_get_next().await.unwrap();
assert_eq!(job.id, job_id);
manager.pause(job_id).await.unwrap();
manager.resume(job_id).await.unwrap();
let (job, _) = manager.try_get_next().await.unwrap();
assert_eq!(job.id, job_id);
}
#[tokio::test]
async fn test_pause_immediately_after_resume() {
let manager = TransferManager::new(4);
let job_id = manager.enqueue_upload("file".into(), "s3/file".into(), None).await;
let (job, _) = manager.try_get_next().await.unwrap();
assert_eq!(job.id, job_id);
manager.pause(job_id).await.unwrap();
manager.resume(job_id).await.unwrap();
manager.pause(job_id).await.unwrap();
assert!(manager.try_get_next().await.is_none());
manager.resume(job_id).await.unwrap();
let (job, _) = manager.try_get_next().await.unwrap();
assert_eq!(job.id, job_id);
}
#[tokio::test]
async fn test_transfer_cancel() {
let manager = TransferManager::new(4);
let job_id = manager.enqueue_upload("file".into(), "s3/file".into(), None).await;
manager.cancel(job_id).await.unwrap();
assert!(manager.try_get_next().await.is_none());
}
#[tokio::test]
async fn test_concurrent_transfer_limit() {
let manager = TransferManager::new(2);
manager.enqueue_upload("file1".into(), "s3/file1".into(), None).await;
manager.enqueue_upload("file2".into(), "s3/file2".into(), None).await;
manager.enqueue_upload("file3".into(), "s3/file3".into(), None).await;
manager.enqueue_upload("file4".into(), "s3/file4".into(), None).await;
assert!(manager.try_get_next().await.is_some());
assert!(manager.try_get_next().await.is_some());
assert!(manager.try_get_next().await.is_none()); }
#[tokio::test]
async fn test_mark_completed() {
let manager = TransferManager::new(4);
let job_id = manager.enqueue_upload("file".into(), "s3/file".into(), None).await;
manager.try_get_next().await;
manager.mark_completed(job_id).await;
manager.enqueue_upload("file2".into(), "s3/file2".into(), None).await;
assert!(manager.try_get_next().await.is_some());
}
#[tokio::test]
async fn test_mark_failed() {
let manager = TransferManager::new(4);
let job_id = manager.enqueue_upload("file".into(), "s3/file".into(), None).await;
manager.try_get_next().await;
manager.mark_failed(job_id, "Network error".into()).await;
manager.enqueue_upload("file2".into(), "s3/file2".into(), None).await;
assert!(manager.try_get_next().await.is_some());
}
}