1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5use tokio::sync::{Mutex, RwLock, mpsc, oneshot};
6use tokio::task::{AbortHandle, Id, JoinSet};
7use tokio::time::interval;
8use tracing::{debug, error, info, warn};
9
10use super::command::{Command, ProgressUpdate};
11use crate::constants;
12use crate::dns::dns_cache::DnsCache;
13use crate::error::{Aria2Error, RecoverableError, Result};
14use crate::ftp::FtpConnectionPool;
15use crate::rate_limiter::{RateLimiter, RateLimiterConfig};
16use crate::request::request_group::RequestGroup;
17use crate::request::request_group_man::RequestGroupMan;
18use crate::retry::{RetryPolicy, RetryStats};
19use crate::session::auto_save_session::AutoSaveSession;
20use crate::session::save_session_command::SaveSessionCommand;
21
22struct RunningTask {
30 handle: AbortHandle,
32 started: Instant,
34 timeout: Option<Duration>,
36}
37
38pub struct DownloadEngine {
39 command_tx: mpsc::UnboundedSender<Box<dyn Command>>,
40 command_rx: mpsc::UnboundedReceiver<Box<dyn Command>>,
41 shutdown_tx: Option<oneshot::Sender<()>>,
42 shutdown_rx: Option<oneshot::Receiver<()>>,
43 tick_interval: Duration,
44 retry_policy: Arc<RetryPolicy>,
45 retry_stats: Arc<RetryStats>,
46 global_limiter: Option<RateLimiter>,
47 save_session_path: Option<PathBuf>,
48 save_session_interval: Option<Duration>,
49 request_group_man: Option<Arc<RwLock<RequestGroupMan>>>,
50 auto_save: Option<Arc<Mutex<AutoSaveSession>>>,
51 ftp_pool: Arc<FtpConnectionPool>,
54 dns_cache: Arc<Mutex<DnsCache>>,
57 keep_alive: bool,
60}
61
62impl DownloadEngine {
63 pub fn new(tick_interval_ms: u64) -> Self {
64 Self::with_retry_policy(tick_interval_ms, RetryPolicy::default())
65 }
66
67 pub fn with_retry_policy(tick_interval_ms: u64, policy: RetryPolicy) -> Self {
68 let (command_tx, command_rx) = mpsc::unbounded_channel();
69 let (shutdown_tx, shutdown_rx) = oneshot::channel();
70
71 let max_tries = policy.max_tries();
72
73 let engine = DownloadEngine {
74 command_tx,
75 command_rx,
76 shutdown_tx: Some(shutdown_tx),
77 shutdown_rx: Some(shutdown_rx),
78 tick_interval: Duration::from_millis(tick_interval_ms),
79 retry_policy: Arc::new(policy),
80 retry_stats: Arc::new(RetryStats::default()),
81 global_limiter: None,
82 save_session_path: None,
83 save_session_interval: None,
84 request_group_man: None,
85 auto_save: None,
86 ftp_pool: Arc::new(FtpConnectionPool::new(
87 constants::FTP_POOL_DEFAULT_MAX_CONNECTIONS,
88 )),
89 dns_cache: Arc::new(Mutex::new(DnsCache::new())),
90 keep_alive: false,
91 };
92
93 info!(
94 "Download engine initialization complete, tick interval: {}ms, max retries: {}",
95 tick_interval_ms, max_tries
96 );
97
98 engine
99 }
100
101 pub fn set_global_rate_limiter(&mut self, config: RateLimiterConfig) {
102 self.global_limiter = Some(RateLimiter::new(&config));
103 info!(
104 "Global speed limits set: download={:?}, upload={:?}",
105 config.download_rate(),
106 config.upload_rate()
107 );
108 }
109
110 pub fn global_rate_limiter(&self) -> Option<&RateLimiter> {
111 self.global_limiter.as_ref()
112 }
113
114 pub fn take_global_rate_limiter(&mut self) -> Option<RateLimiter> {
115 self.global_limiter.take()
116 }
117
118 pub fn spawn_progress_aggregator(
141 group: Arc<RwLock<RequestGroup>>,
142 mut receiver: mpsc::UnboundedReceiver<ProgressUpdate>,
143 ) -> tokio::task::JoinHandle<()> {
144 tokio::spawn(async move {
145 let mut last_bytes: u64 = 0;
146 while let Some(update) = receiver.recv().await {
147 if update.completed_bytes == last_bytes {
150 continue;
151 }
152 let g = group.write().await;
153 g.update_progress(update.completed_bytes).await;
154 g.set_completed_length(update.completed_bytes);
155 if update.download_speed > 0 {
156 g.update_speed(update.download_speed, update.upload_speed)
157 .await;
158 g.set_download_speed_cached(update.download_speed);
159 }
160 last_bytes = update.completed_bytes;
161 }
162 })
163 }
164
165 pub fn set_save_session(
166 &mut self,
167 path: PathBuf,
168 interval: Option<Duration>,
169 man: Arc<RwLock<RequestGroupMan>>,
170 ) {
171 self.save_session_path = Some(path.clone());
172 self.save_session_interval = interval;
173 self.request_group_man = Some(man);
174
175 if let (Some(interval), Some(man_ref)) = (interval, &self.request_group_man) {
176 let path_clone = path.clone();
177 let auto_save = AutoSaveSession::new(path, interval, man_ref.clone());
178 self.auto_save = Some(Arc::new(Mutex::new(auto_save)));
179 info!(
180 "Auto-save session enabled: path={}, interval={:.1}s",
181 path_clone.display(),
182 interval.as_secs_f64()
183 );
184 } else {
185 info!("Manual save session enabled: path={}", path.display());
186 }
187 }
188
189 pub fn mark_session_dirty(&self) {
190 if let Some(ref auto_save) = self.auto_save
191 && let Ok(auto) = auto_save.try_lock()
192 {
193 auto.mark_dirty();
194 }
195 }
196
197 pub fn save_session_path(&self) -> Option<&PathBuf> {
198 self.save_session_path.as_ref()
199 }
200
201 pub fn add_command(&self, command: Box<dyn Command>) -> Result<()> {
202 self.command_tx
203 .send(command)
204 .map_err(|e| Aria2Error::DownloadFailed(format!("Failed to add command: {}", e)))
205 }
206
207 pub fn retry_stats(&self) -> &RetryStats {
208 &self.retry_stats
209 }
210
211 pub fn retry_policy(&self) -> &RetryPolicy {
212 &self.retry_policy
213 }
214
215 pub fn ftp_pool(&self) -> &Arc<FtpConnectionPool> {
217 &self.ftp_pool
218 }
219
220 pub fn dns_cache(&self) -> &Arc<Mutex<DnsCache>> {
222 &self.dns_cache
223 }
224
225 pub fn set_keep_alive(&mut self, v: bool) {
229 self.keep_alive = v;
230 }
231
232 pub fn command_sender(&self) -> mpsc::UnboundedSender<Box<dyn Command>> {
235 self.command_tx.clone()
236 }
237
238 pub fn take_shutdown_sender(&mut self) -> Option<oneshot::Sender<()>> {
241 self.shutdown_tx.take()
242 }
243
244 pub async fn run(mut self) -> Result<()> {
245 info!("Download engine started");
246
247 let mut pending_commands: Vec<Box<dyn Command>> = Vec::new();
248 let mut running: JoinSet<Result<()>> = JoinSet::new();
254 let mut running_tasks: HashMap<Id, RunningTask> = HashMap::new();
255 let mut failed_commands: Vec<(Box<dyn Command>, u32)> = Vec::new();
259
260 let mut ticker = interval(self.tick_interval);
261 let mut shutdown_rx = self
262 .shutdown_rx
263 .take()
264 .expect("shutdown_rx should exist in run()");
265 let policy = self.retry_policy.clone();
266 let stats = self.retry_stats.clone();
267
268 loop {
269 tokio::select! {
270 _ = ticker.tick() => {
271 debug!("Engine tick triggered");
272
273 for (cmd, attempt) in failed_commands.drain(..) {
275 if policy.should_retry(attempt, &Aria2Error::Recoverable(RecoverableError::Timeout)) {
276 let wait = policy.wait_duration(attempt);
277 warn!("Retrying command (attempt {}), waiting {:?}", attempt + 1, wait);
278 pending_commands.push(cmd);
279 tokio::time::sleep(wait).await;
280 } else {
281 error!("Command retry abandoned (attempted {} times)", attempt + 1);
282 }
283 }
284
285 self.dispatch_commands(
287 &mut pending_commands,
288 &mut running,
289 &mut running_tasks,
290 )
291 .await?;
292
293 self.check_timeouts(&mut running_tasks, &stats).await?;
295
296 self.collect_completed(&mut running, &mut running_tasks).await?;
298
299 if !self.keep_alive
304 && pending_commands.is_empty()
305 && running.is_empty()
306 && failed_commands.is_empty()
307 {
308 info!("All tasks completed, engine shutting down");
309 break;
310 }
311 }
312
313 Ok(_) = &mut shutdown_rx => {
314 info!("Shutdown signal received");
315 self.shutdown(&mut running).await;
316 break;
317 }
318 }
319 }
320
321 info!(
322 "Download engine stopped, retry stats: total={}, timeouts={}, server_errors={}, network_failures={}",
323 stats.total(),
324 stats.timeouts(),
325 stats.server_errors(),
326 stats.network_failures()
327 );
328 Ok(())
329 }
330
331 async fn dispatch_commands(
339 &mut self,
340 pending: &mut Vec<Box<dyn Command>>,
341 running: &mut JoinSet<Result<()>>,
342 running_tasks: &mut HashMap<Id, RunningTask>,
343 ) -> Result<()> {
344 while let Ok(cmd) = self.command_rx.try_recv() {
346 pending.push(cmd);
347 }
348
349 while !pending.is_empty() {
350 let mut cmd = pending.remove(0);
351 cmd.set_started_at(Instant::now());
354 let timeout = cmd.timeout();
355
356 let handle = running.spawn(async move {
360 let mut cmd = cmd;
361 cmd.execute().await
362 });
363 let id = handle.id();
364 running_tasks.insert(
365 id,
366 RunningTask {
367 handle,
368 started: Instant::now(),
369 timeout,
370 },
371 );
372 debug!(
373 "Dispatched command (task {}), running: {}",
374 id,
375 running.len()
376 );
377 }
378 Ok(())
379 }
380
381 async fn check_timeouts(
396 &self,
397 running_tasks: &mut HashMap<Id, RunningTask>,
398 stats: &RetryStats,
399 ) -> Result<()> {
400 let now = Instant::now();
401 let timed_out: Vec<Id> = running_tasks
404 .iter()
405 .filter_map(|(id, task)| {
406 let dur = task.timeout?;
407 if now.duration_since(task.started) > dur {
408 Some(*id)
409 } else {
410 None
411 }
412 })
413 .collect();
414
415 for id in timed_out {
416 if let Some(task) = running_tasks.remove(&id) {
417 warn!("Command (task {}) timed out, aborting", id);
418 task.handle.abort();
422 stats.record_retry(&Aria2Error::Recoverable(RecoverableError::Timeout));
423 }
424 }
425 Ok(())
426 }
427
428 async fn collect_completed(
435 &self,
436 running: &mut JoinSet<Result<()>>,
437 running_tasks: &mut HashMap<Id, RunningTask>,
438 ) -> Result<()> {
439 while let Some(join_result) = running.try_join_next_with_id() {
440 match join_result {
441 Ok((id, Ok(()))) => {
442 running_tasks.remove(&id);
443 debug!("Command (task {}) completed successfully", id);
444 }
445 Ok((id, Err(e))) => {
446 running_tasks.remove(&id);
447 error!("Command (task {}) execution failed: {}", id, e);
451 }
452 Err(join_err) => {
453 let id = join_err.id();
454 running_tasks.remove(&id);
455 if join_err.is_cancelled() {
456 debug!("Command (task {}) was cancelled", id);
458 } else if join_err.is_panic() {
459 error!("Command (task {}) panicked: {}", id, join_err);
460 } else {
461 warn!("Command (task {}) joined with error: {}", id, join_err);
462 }
463 }
464 }
465 }
466 Ok(())
467 }
468
469 async fn shutdown(&self, running: &mut JoinSet<Result<()>>) {
470 info!("Shutting down running commands...");
471 if let (Some(path), Some(man)) = (&self.save_session_path, &self.request_group_man) {
474 let mut cmd = SaveSessionCommand::new(path.clone(), man.clone());
475 match cmd.execute().await {
476 Ok(_) => info!("Session saved on shutdown to {}", path.display()),
477 Err(e) => warn!("Failed to save session on shutdown: {}", e),
478 }
479 }
480 running.abort_all();
482 }
483
484 pub async fn shutdown_engine(&mut self) -> Result<()> {
485 if let Some(tx) = self.shutdown_tx.take() {
486 let _ = tx.send(());
487 }
488 Ok(())
489 }
490}
491
492#[cfg(test)]
493mod tests {
494 use super::*;
495 use crate::engine::command::CommandStatus;
496 use async_trait::async_trait;
497 use std::sync::Arc;
498 use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
499 use std::time::{Duration, Instant};
500
501 struct StalledCommand {
505 timeout_dur: Duration,
506 completed: Arc<AtomicBool>,
507 }
508
509 #[async_trait]
510 impl Command for StalledCommand {
511 async fn execute(&mut self) -> Result<()> {
512 tokio::time::sleep(Duration::from_secs(30)).await;
514 self.completed.store(true, Ordering::SeqCst);
515 Ok(())
516 }
517
518 fn status(&self) -> CommandStatus {
519 CommandStatus::Running
520 }
521
522 fn timeout(&self) -> Option<Duration> {
523 Some(self.timeout_dur)
524 }
525 }
526
527 #[tokio::test]
533 async fn test_check_timeouts_actually_times_out_stalled_command() {
534 let engine = DownloadEngine::new(10);
536 let stats = engine.retry_stats.clone();
539
540 let completed = Arc::new(AtomicBool::new(false));
541 let cmd = StalledCommand {
542 timeout_dur: Duration::from_millis(50),
543 completed: completed.clone(),
544 };
545 engine.add_command(Box::new(cmd)).unwrap();
546
547 let run_result = tokio::time::timeout(Duration::from_secs(2), engine.run()).await;
551
552 assert!(
553 run_result.is_ok(),
554 "engine.run() should complete within 2s, not hang (timeout not enforced?)"
555 );
556 assert!(
557 stats.timeouts() >= 1,
558 "expected at least 1 recorded timeout, got {} (Task A1 bug: empty future)",
559 stats.timeouts()
560 );
561 assert!(
562 !completed.load(Ordering::SeqCst),
563 "stalled command should have been aborted before completing"
564 );
565 }
566
567 struct SlowCommand {
570 delay: Duration,
571 completed: Arc<AtomicU32>,
572 }
573
574 #[async_trait]
575 impl Command for SlowCommand {
576 async fn execute(&mut self) -> Result<()> {
577 tokio::time::sleep(self.delay).await;
578 self.completed.fetch_add(1, Ordering::SeqCst);
579 Ok(())
580 }
581
582 fn status(&self) -> CommandStatus {
583 CommandStatus::Running
584 }
585 }
586
587 #[tokio::test]
592 async fn test_engine_dispatches_commands_concurrently() {
593 let engine = DownloadEngine::new(10);
595 let completed = Arc::new(AtomicU32::new(0));
596 for _ in 0..5 {
597 let cmd = SlowCommand {
598 delay: Duration::from_millis(100),
599 completed: completed.clone(),
600 };
601 engine.add_command(Box::new(cmd)).unwrap();
602 }
603
604 let start = Instant::now();
605 let run_result = tokio::time::timeout(Duration::from_millis(500), engine.run()).await;
606 let elapsed = start.elapsed();
607
608 assert!(
609 run_result.is_ok(),
610 "engine.run() should complete within 500ms, not hang"
611 );
612 assert_eq!(
613 completed.load(Ordering::SeqCst),
614 5,
615 "all 5 commands should have completed"
616 );
617 assert!(
621 elapsed < Duration::from_millis(300),
622 "5x100ms commands took {:?}, indicating serialization (concurrent should be ~150ms)",
623 elapsed
624 );
625 }
626
627 use crate::engine::command::ProgressUpdate;
630 use crate::request::request_group::{DownloadOptions, GroupId, RequestGroup};
631
632 fn make_group() -> Arc<RwLock<RequestGroup>> {
635 Arc::new(RwLock::new(RequestGroup::new(
636 GroupId::new(1),
637 vec!["http://example.com/file.bin".to_string()],
638 DownloadOptions::default(),
639 )))
640 }
641
642 #[tokio::test]
647 async fn test_progress_channel_updates_request_group() {
648 let group = make_group();
649 let (tx, rx) = mpsc::unbounded_channel::<ProgressUpdate>();
650
651 let group_clone = Arc::clone(&group);
652 let handle = DownloadEngine::spawn_progress_aggregator(group_clone, rx);
653
654 tx.send(ProgressUpdate {
656 completed_bytes: 1000,
657 download_speed: 0,
658 upload_speed: 0,
659 })
660 .unwrap();
661 tx.send(ProgressUpdate {
662 completed_bytes: 5000,
663 download_speed: 0,
664 upload_speed: 0,
665 })
666 .unwrap();
667
668 drop(tx);
673 handle.await.expect("aggregator task should exit cleanly");
674
675 let atomic_val = { group.read().await.get_completed_length() };
677 assert_eq!(
678 atomic_val, 5000,
679 "aggregator should have applied the latest completed_bytes (5000)"
680 );
681 }
682
683 #[tokio::test]
687 async fn test_progress_aggregator_dedupes_identical_bytes() {
688 let group = make_group();
689 let (tx, rx) = mpsc::unbounded_channel::<ProgressUpdate>();
690
691 let group_clone = Arc::clone(&group);
692 let handle = DownloadEngine::spawn_progress_aggregator(group_clone, rx);
693
694 tx.send(ProgressUpdate {
696 completed_bytes: 2048,
697 download_speed: 0,
698 upload_speed: 0,
699 })
700 .unwrap();
701 for _ in 0..5 {
705 tx.send(ProgressUpdate {
706 completed_bytes: 2048,
707 download_speed: 9999,
708 upload_speed: 0,
709 })
710 .unwrap();
711 }
712 tx.send(ProgressUpdate {
714 completed_bytes: 4096,
715 download_speed: 1234,
716 upload_speed: 0,
717 })
718 .unwrap();
719
720 drop(tx);
723 handle.await.expect("aggregator task should exit cleanly");
724
725 let g = group.read().await;
726 assert_eq!(
727 g.get_completed_length(),
728 4096,
729 "final completed_bytes should be 4096"
730 );
731 assert_eq!(
732 g.get_download_speed_cached(),
733 1234,
734 "speed from the advancing update should be cached"
735 );
736 }
737
738 #[tokio::test]
742 async fn test_progress_aggregator_preserves_speed_on_zero_sample() {
743 let group = make_group();
744 let (tx, rx) = mpsc::unbounded_channel::<ProgressUpdate>();
745
746 {
748 let g = group.read().await;
749 g.set_download_speed_cached(5555);
750 }
751
752 let group_clone = Arc::clone(&group);
753 let handle = DownloadEngine::spawn_progress_aggregator(group_clone, rx);
754
755 tx.send(ProgressUpdate {
757 completed_bytes: 8192,
758 download_speed: 0,
759 upload_speed: 0,
760 })
761 .unwrap();
762
763 drop(tx);
765 handle.await.expect("aggregator task should exit cleanly");
766
767 let g = group.read().await;
768 assert_eq!(g.get_completed_length(), 8192, "bytes should advance");
769 assert_eq!(
770 g.get_download_speed_cached(),
771 5555,
772 "cached speed must be preserved when download_speed sample is 0"
773 );
774 }
775
776 #[tokio::test]
779 async fn test_progress_aggregator_exits_on_sender_drop() {
780 let group = make_group();
781 let (tx, rx) = mpsc::unbounded_channel::<ProgressUpdate>();
782
783 let handle = DownloadEngine::spawn_progress_aggregator(group, rx);
784
785 drop(tx);
787
788 let result = tokio::time::timeout(Duration::from_millis(500), handle).await;
790 assert!(
791 result.is_ok(),
792 "aggregator should exit within 500ms after senders are dropped"
793 );
794 result
795 .expect("aggregator task should exit cleanly")
796 .expect("aggregator task should not panic");
797 }
798}