aria2-protocol 0.2.8

Multi-protocol networking stack for aria2-rust: HTTP/HTTPS client, FTP/SFTP, full BitTorrent (DHT/PEX/MSE), and Metalink V3/V4 parser
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
//! DHT Task scheduling — async task executor with concurrency-limited queues.
//!
//! This module provides the Rust equivalent of the C++ DHT task scheduling
//! system (`DHTTask`, `DHTTaskExecutor`, `DHTTaskQueue`, `DHTTaskFactory`).
//!
//! Key differences from the C++ design:
//! - C++ uses synchronous `startup()` + callback-based `finished()` checks.
//!   Rust uses `async fn run()` that returns when the task completes.
//! - C++ polls `update()` every event-loop iteration. Rust uses `tokio::spawn`
//!   with a semaphore for concurrency control, so tasks run independently.
//! - C++ has three fixed priority executors (periodicTaskQueue1/2 + immediate).
//!   Rust preserves the same three-lane design via `DhtTaskQueue`.
//!
//! Architecture:
//! ```text
//! DhtTaskQueue
//!   ├── periodic_executor_1: DhtTaskExecutor (bucket refresh, node lookup)
//!   ├── periodic_executor_2: DhtTaskExecutor (keep-alive pings)
//!   └── immediate_executor:  DhtTaskExecutor (announce, on-demand lookup)
//!
//! DhtTaskExecutor
//!   ├── queue: VecDeque<BoxedDhtTask>   (pending tasks)
//!   └── semaphore: Semaphore(num_concurrent) (limits in-flight tasks)
//! ```

use std::collections::VecDeque;
use std::fmt;
use std::sync::Arc;

use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore};
use tracing::{debug, trace};

// ---------------------------------------------------------------------------
// DhtTask trait
// ---------------------------------------------------------------------------

/// Core trait for all DHT tasks.
///
/// Equivalent to C++ `DHTTask` with `startup()` and `finished()`, but
/// expressed as a single `async fn run()` that completes when the task
/// is done. The executor manages concurrency and scheduling externally.
#[async_trait::async_trait]
pub trait DhtTask: Send + fmt::Debug {
    /// Execute the task to completion.
    ///
    /// Implementations should perform their work and return when finished.
    /// The executor ensures that no more than `num_concurrent` tasks run
    /// simultaneously within each priority lane.
    async fn run(self: Box<Self>);

    /// Human-readable task name for logging.
    fn name(&self) -> &'static str;
}

/// Type-erased boxed DHT task.
pub type BoxedDhtTask = Box<dyn DhtTask>;

// ---------------------------------------------------------------------------
// DhtTaskExecutor
// ---------------------------------------------------------------------------

/// Default concurrency limit per executor lane (matches C++ `NUM_CONCURRENT_TASK = 15`).
pub const DEFAULT_NUM_CONCURRENT: usize = 15;

/// A concurrency-limited executor for DHT tasks.
///
/// Equivalent to C++ `DHTTaskExecutor`. Tasks are queued and dispatched
/// up to `num_concurrent` at a time. When a task finishes, the next
/// queued task is started.
///
/// Unlike the C++ version which is polled synchronously via `update()`,
/// this Rust version uses a `tokio::sync::Semaphore` and spawns tasks
/// via `tokio::spawn` so that they run concurrently without blocking
/// the executor's dispatch loop.
pub struct DhtTaskExecutor {
    /// Shared state protected by an async mutex.
    inner: Arc<Mutex<DhtTaskExecutorInner>>,
    /// Semaphore controlling maximum concurrency.
    semaphore: Arc<Semaphore>,
    /// Maximum concurrent tasks.
    num_concurrent: usize,
}

struct DhtTaskExecutorInner {
    /// FIFO queue of pending tasks.
    queue: VecDeque<BoxedDhtTask>,
    /// Number of currently executing tasks.
    executing: usize,
}

impl DhtTaskExecutor {
    /// Create a new executor with the given concurrency limit.
    pub fn new(num_concurrent: usize) -> Self {
        Self {
            inner: Arc::new(Mutex::new(DhtTaskExecutorInner {
                queue: VecDeque::new(),
                executing: 0,
            })),
            semaphore: Arc::new(Semaphore::new(num_concurrent)),
            num_concurrent,
        }
    }

    /// Enqueue a task for execution.
    ///
    /// If there is capacity, the task is dispatched immediately.
    /// Otherwise it waits in the FIFO queue until a slot opens.
    pub async fn add_task(&self, task: BoxedDhtTask) {
        let task_name = task.name();
        let mut inner = self.inner.lock().await;
        inner.queue.push_back(task);
        trace!(
            task = task_name,
            queue_len = inner.queue.len(),
            executing = inner.executing,
            "DHT task enqueued"
        );
        drop(inner);

        // Try to dispatch pending tasks.
        self.dispatch_pending().await;
    }

    /// Number of currently executing tasks.
    pub async fn executing_count(&self) -> usize {
        self.inner.lock().await.executing
    }

    /// Number of tasks waiting in the queue.
    pub async fn queue_size(&self) -> usize {
        self.inner.lock().await.queue.len()
    }

    /// Try to dispatch as many queued tasks as the semaphore allows.
    async fn dispatch_pending(&self) {
        let sem = Arc::clone(&self.semaphore);
        loop {
            let task = {
                let mut inner = self.inner.lock().await;
                if inner.queue.is_empty() {
                    break;
                }
                inner.queue.pop_front()
            };

            let Some(task) = task else { break };
            let task_name = task.name();

            // Try non-blocking permit acquisition on the owned semaphore.
            match sem.clone().try_acquire_owned() {
                Ok(permit) => {
                    let mut inner = self.inner.lock().await;
                    inner.executing += 1;
                    drop(inner);

                    debug!(task = task_name, "DHT task dispatched");

                    // Spawn the task, holding the owned permit until done.
                    self.spawn_task(task, permit);
                }
                Err(_) => {
                    // At capacity — re-queue the task at the front and stop.
                    let mut inner = self.inner.lock().await;
                    inner.queue.push_front(task);
                    break;
                }
            }
        }
    }

    /// Spawn a task on the tokio runtime, holding the owned semaphore permit
    /// for the task's lifetime. When the task completes, the permit is
    /// automatically released (via `Drop`), allowing the next queued task
    /// to be dispatched.
    fn spawn_task(&self, task: BoxedDhtTask, _permit: OwnedSemaphorePermit) {
        let inner = Arc::clone(&self.inner);
        let semaphore = Arc::clone(&self.semaphore);

        // The core task runner: runs the task, then re-dispatches.
        // This function returns a Future that the spawner awaits.
        let run_and_redispatch = async move {
            // Hold the permit for the duration of the task.
            let _held = _permit;

            task.run().await;

            // Task completed — update executing count.
            {
                let mut guard = inner.lock().await;
                guard.executing = guard.executing.saturating_sub(1);
            }

            // Release the permit so the next task can start.
            drop(_held);

            // Re-dispatch any pending tasks now that a slot is free.
            loop {
                let next_task = {
                    let mut guard = inner.lock().await;
                    guard.queue.pop_front()
                };

                let Some(next_task) = next_task else { break };

                match semaphore.clone().try_acquire_owned() {
                    Ok(permit) => {
                        {
                            let mut guard = inner.lock().await;
                            guard.executing += 1;
                        }
                        debug!(
                            task = next_task.name(),
                            "DHT task dispatched (after completion)"
                        );

                        // Recursively run the next task in this same
                        // coroutine. This avoids spawning unlimited
                        // tasks and ensures the chain continues.
                        // The permit is held across the recursive call.
                        let _held2 = permit;
                        next_task.run().await;

                        {
                            let mut guard = inner.lock().await;
                            guard.executing = guard.executing.saturating_sub(1);
                        }
                        drop(_held2);
                        // Loop continues — try to dispatch more.
                    }
                    Err(_) => {
                        // No permits available — re-queue and stop.
                        let mut guard = inner.lock().await;
                        guard.queue.push_front(next_task);
                        break;
                    }
                }
            }
        };

        tokio::spawn(run_and_redispatch);
    }
}

impl fmt::Debug for DhtTaskExecutor {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("DhtTaskExecutor")
            .field("num_concurrent", &self.num_concurrent)
            .finish()
    }
}

// ---------------------------------------------------------------------------
// DhtTaskQueue — three-lane priority queue
// ---------------------------------------------------------------------------

/// Three-lane DHT task queue, equivalent to C++ `DHTTaskQueueImpl`.
///
/// - **Periodic 1**: Bucket refresh and node lookup tasks (higher priority).
/// - **Periodic 2**: Keep-alive pings and maintenance tasks.
/// - **Immediate**: On-demand tasks triggered by user actions (announce,
///   peer lookup, etc.).
///
/// Each lane has its own `DhtTaskExecutor` with an independent concurrency
/// limit, so that immediate tasks are never starved by periodic maintenance.
pub struct DhtTaskQueue {
    /// Periodic lane 1: bucket refresh, node lookup (C++ periodicTaskQueue1_).
    periodic_executor_1: DhtTaskExecutor,

    /// Periodic lane 2: keep-alive pings, maintenance (C++ periodicTaskQueue2_).
    periodic_executor_2: DhtTaskExecutor,

    /// Immediate lane: on-demand user tasks (C++ immediateTaskQueue_).
    immediate_executor: DhtTaskExecutor,
}

impl DhtTaskQueue {
    /// Create a new task queue with the default concurrency limit.
    pub fn new() -> Self {
        Self::with_concurrency(DEFAULT_NUM_CONCURRENT)
    }

    /// Create a new task queue with a custom concurrency limit.
    pub fn with_concurrency(num_concurrent: usize) -> Self {
        Self {
            periodic_executor_1: DhtTaskExecutor::new(num_concurrent),
            periodic_executor_2: DhtTaskExecutor::new(num_concurrent),
            immediate_executor: DhtTaskExecutor::new(num_concurrent),
        }
    }

    /// Add a task to periodic lane 1 (bucket refresh, node lookup).
    ///
    /// Equivalent to C++ `DHTTaskQueueImpl::addPeriodicTask1()`.
    pub async fn add_periodic_task_1(&self, task: BoxedDhtTask) {
        self.periodic_executor_1.add_task(task).await;
    }

    /// Add a task to periodic lane 2 (keep-alive, maintenance).
    ///
    /// Equivalent to C++ `DHTTaskQueueImpl::addPeriodicTask2()`.
    pub async fn add_periodic_task_2(&self, task: BoxedDhtTask) {
        self.periodic_executor_2.add_task(task).await;
    }

    /// Add an immediate (on-demand) task.
    ///
    /// Equivalent to C++ `DHTTaskQueueImpl::addImmediateTask()`.
    pub async fn add_immediate_task(&self, task: BoxedDhtTask) {
        self.immediate_executor.add_task(task).await;
    }

    /// Reference to periodic executor 1 (for direct access if needed).
    pub fn periodic_executor_1(&self) -> &DhtTaskExecutor {
        &self.periodic_executor_1
    }

    /// Reference to periodic executor 2.
    pub fn periodic_executor_2(&self) -> &DhtTaskExecutor {
        &self.periodic_executor_2
    }

    /// Reference to immediate executor.
    pub fn immediate_executor(&self) -> &DhtTaskExecutor {
        &self.immediate_executor
    }
}

impl Default for DhtTaskQueue {
    fn default() -> Self {
        Self::new()
    }
}

impl fmt::Debug for DhtTaskQueue {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("DhtTaskQueue").finish()
    }
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};

    /// A trivial test task that increments a counter.
    #[derive(Debug)]
    struct CountTask {
        counter: Arc<AtomicUsize>,
        name: &'static str,
    }

    #[async_trait::async_trait]
    impl DhtTask for CountTask {
        async fn run(self: Box<Self>) {
            self.counter.fetch_add(1, Ordering::SeqCst);
        }

        fn name(&self) -> &'static str {
            self.name
        }
    }

    #[tokio::test]
    async fn test_executor_dispatches_task() {
        let executor = DhtTaskExecutor::new(2);
        let counter = Arc::new(AtomicUsize::new(0));

        executor
            .add_task(Box::new(CountTask {
                counter: Arc::clone(&counter),
                name: "test",
            }))
            .await;

        // Give the spawned task time to complete.
        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

        assert_eq!(counter.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn test_executor_concurrency_limit() {
        let executor = DhtTaskExecutor::new(2);
        let counter = Arc::new(AtomicUsize::new(0));

        // Enqueue 5 tasks — only 2 should execute concurrently.
        for _ in 0..5 {
            executor
                .add_task(Box::new(CountTask {
                    counter: Arc::clone(&counter),
                    name: "concurrent-test",
                }))
                .await;
        }

        // Wait for all tasks to complete.
        tokio::time::sleep(std::time::Duration::from_millis(200)).await;

        assert_eq!(counter.load(Ordering::SeqCst), 5);
        assert_eq!(executor.executing_count().await, 0);
    }

    #[tokio::test]
    async fn test_task_queue_three_lanes() {
        let queue = DhtTaskQueue::new();
        let c1 = Arc::new(AtomicUsize::new(0));
        let c2 = Arc::new(AtomicUsize::new(0));
        let c3 = Arc::new(AtomicUsize::new(0));

        queue
            .add_periodic_task_1(Box::new(CountTask {
                counter: Arc::clone(&c1),
                name: "p1",
            }))
            .await;
        queue
            .add_periodic_task_2(Box::new(CountTask {
                counter: Arc::clone(&c2),
                name: "p2",
            }))
            .await;
        queue
            .add_immediate_task(Box::new(CountTask {
                counter: Arc::clone(&c3),
                name: "imm",
            }))
            .await;

        tokio::time::sleep(std::time::Duration::from_millis(100)).await;

        assert_eq!(c1.load(Ordering::SeqCst), 1);
        assert_eq!(c2.load(Ordering::SeqCst), 1);
        assert_eq!(c3.load(Ordering::SeqCst), 1);
    }

    #[tokio::test]
    async fn test_executor_queue_size() {
        let executor = DhtTaskExecutor::new(1);
        let counter = Arc::new(AtomicUsize::new(0));

        // A slow task that holds the slot.
        #[derive(Debug)]
        struct SlowTask {
            counter: Arc<AtomicUsize>,
        }

        #[async_trait::async_trait]
        impl DhtTask for SlowTask {
            async fn run(self: Box<Self>) {
                tokio::time::sleep(std::time::Duration::from_millis(100)).await;
                self.counter.fetch_add(1, Ordering::SeqCst);
            }

            fn name(&self) -> &'static str {
                "slow"
            }
        }

        executor
            .add_task(Box::new(SlowTask {
                counter: Arc::clone(&counter),
            }))
            .await;

        // Add more tasks while the first is running.
        for _ in 0..3 {
            executor
                .add_task(Box::new(CountTask {
                    counter: Arc::clone(&counter),
                    name: "queued",
                }))
                .await;
        }

        // Should have 1 executing + some queued.
        let executing = executor.executing_count().await;
        let queued = executor.queue_size().await;
        assert!(
            executing + queued >= 3,
            "executing={}, queued={}",
            executing,
            queued
        );

        // Wait for all to finish.
        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
        assert_eq!(counter.load(Ordering::SeqCst), 4);
    }
}