aria2-core 0.3.10

aria2-rust core download engine: multi-protocol segmented downloads, rate limiting, configuration, session persistence, and BitTorrent seeding
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
//! Rust-owned BitTorrent piece checkpoints.
//!
//! The public sidecar path remains the familiar `.aria2` location. New bytes
//! use aria2's native control-file layout plus the small project extension
//! needed for verified-piece metadata; the retired A2CF reader is migration
//! compatibility only.

use std::path::{Path, PathBuf};

use crate::error::Result;
use crate::filesystem::control_file::{ControlFile, ControlFileInFlightPiece};

pub(crate) struct BtCheckpoint {
    control_file: Option<ControlFile>,
    path: PathBuf,
    total_length: u64,
    piece_length: u32,
    num_pieces: usize,
    info_hash: [u8; 20],
}

impl BtCheckpoint {
    pub(crate) async fn open(
        output_path: &Path,
        payload_exists: bool,
        total_length: u64,
        piece_length: u32,
        num_pieces: usize,
        info_hash: [u8; 20],
    ) -> Result<Self> {
        let path = ControlFile::control_path_for(output_path);
        let expected_bitfield_len = num_pieces.div_ceil(8);
        let control_file = match ControlFile::load(&path).await {
            Ok(Some(mut control_file))
                if control_file.is_torrent_checkpoint()
                    && control_file.total_length() == total_length
                    && payload_exists
                    && control_file.bitfield().len() == expected_bitfield_len
                    && valid_trailing_bits(control_file.bitfield(), num_pieces)
                    && control_file.torrent_info_hash() == Some(info_hash)
                    && control_file
                        .torrent_piece_length()
                        .is_none_or(|stored| stored == piece_length) =>
            {
                // Legacy checkpoints written before piece-length persistence
                // are still safe to restore because the torrent info hash
                // identifies the complete info dictionary. Normalize their
                // byte-only bitfield before the next native-format save.
                control_file.normalize_legacy_piece_layout(num_pieces)?;
                control_file.set_torrent_piece_length(piece_length);
                Some(control_file)
            }
            Ok(Some(_)) => {
                remove_stale(&path).await;
                None
            }
            Ok(None) => None,
            Err(error) => {
                tracing::debug!(
                    path = %path.display(),
                    %error,
                    "Ignoring unreadable BitTorrent checkpoint"
                );
                remove_stale(&path).await;
                None
            }
        };

        let mut checkpoint = Self {
            control_file,
            path,
            total_length,
            piece_length,
            num_pieces,
            info_hash,
        };
        if checkpoint.control_file.is_none() {
            checkpoint.control_file = Some(
                ControlFile::open_or_create(&checkpoint.path, total_length, num_pieces).await?,
            );
            if let Some(control_file) = checkpoint.control_file.as_mut() {
                control_file.mark_torrent_checkpoint();
                control_file.set_torrent_info_hash(info_hash);
                control_file.set_torrent_piece_length(piece_length);
            }
        }
        Ok(checkpoint)
    }

    pub(crate) fn bitfield(&self) -> Option<&[u8]> {
        self.control_file
            .as_ref()
            .map(ControlFile::bitfield)
            .filter(|bitfield| bitfield.len() == self.num_pieces.div_ceil(8))
            .filter(|bitfield| valid_trailing_bits(bitfield, self.num_pieces))
    }

    pub(crate) fn completed_length(&self) -> u64 {
        let Some(bitfield) = self.bitfield() else {
            return 0;
        };

        (0..self.num_pieces)
            .filter(|&index| bit_is_set(bitfield, index))
            .map(|index| {
                let offset = index as u64 * self.piece_length as u64;
                self.total_length
                    .saturating_sub(offset)
                    .min(self.piece_length as u64)
            })
            .sum()
    }

    pub(crate) fn in_flight_pieces(&self) -> &[ControlFileInFlightPiece] {
        self.control_file
            .as_ref()
            .map(ControlFile::in_flight_pieces)
            .unwrap_or_default()
    }

    pub(crate) async fn save(&mut self, bitfield: &[u8], completed_length: u64) -> Result<()> {
        self.save_with_in_flight_pieces(bitfield, completed_length, &[])
            .await
    }

    pub(crate) async fn save_with_in_flight_pieces(
        &mut self,
        bitfield: &[u8],
        completed_length: u64,
        in_flight_pieces: &[ControlFileInFlightPiece],
    ) -> Result<()> {
        if bitfield.len() != self.num_pieces.div_ceil(8) {
            return Err(crate::error::Aria2Error::FileIo(format!(
                "BitTorrent checkpoint bitfield length mismatch: expected {}, got {}",
                self.num_pieces.div_ceil(8),
                bitfield.len()
            )));
        }
        if !valid_trailing_bits(bitfield, self.num_pieces) {
            return Err(crate::error::Aria2Error::FileIo(
                "BitTorrent checkpoint contains set trailing bits".into(),
            ));
        }
        let control_file = self.control_file.as_mut().ok_or_else(|| {
            crate::error::Aria2Error::FileIo("BitTorrent checkpoint is unavailable".into())
        })?;
        control_file.mark_torrent_checkpoint();
        if control_file.torrent_info_hash() != Some(self.info_hash) {
            return Err(crate::error::Aria2Error::FileIo(
                "BitTorrent checkpoint torrent identity mismatch".into(),
            ));
        }
        if control_file.torrent_piece_length() != Some(self.piece_length) {
            return Err(crate::error::Aria2Error::FileIo(
                "BitTorrent checkpoint piece length mismatch".into(),
            ));
        }
        // Keep the in-memory snapshot unchanged until the durable replacement
        // succeeds. A failed disk write must not make this instance appear
        // newer than the checkpoint that can actually be restored.
        let mut next = control_file.clone();
        next.set_bitfield(bitfield.to_vec());
        next.set_in_flight_pieces(in_flight_pieces.to_vec());
        next.update_completed_length(completed_length.min(self.total_length));
        next.save().await?;
        *control_file = next;
        Ok(())
    }

    pub(crate) async fn save_verified_pieces<I>(
        &mut self,
        indices: I,
        completed_length: u64,
    ) -> Result<()>
    where
        I: IntoIterator<Item = usize>,
    {
        let mut bitfield = vec![0u8; self.num_pieces.div_ceil(8)];
        for index in indices {
            if index < self.num_pieces {
                bitfield[index / 8] |= 1 << (7 - index % 8);
            }
        }
        self.save(&bitfield, completed_length).await
    }

    pub(crate) async fn remove(self) -> Result<()> {
        match tokio::fs::remove_file(&self.path).await {
            Ok(()) => Ok(()),
            Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
            Err(error) => Err(crate::error::Aria2Error::FileIo(format!(
                "Failed to remove BitTorrent checkpoint {}: {error}",
                self.path.display()
            ))),
        }
    }
}

fn bit_is_set(bitfield: &[u8], index: usize) -> bool {
    bitfield
        .get(index / 8)
        .is_some_and(|byte| byte & (1 << (7 - index % 8)) != 0)
}

fn valid_trailing_bits(bitfield: &[u8], num_pieces: usize) -> bool {
    if num_pieces == 0 {
        return bitfield.is_empty();
    }
    let unused_bits = (8 - num_pieces % 8) % 8;
    unused_bits == 0
        || bitfield
            .last()
            .is_some_and(|byte| byte & ((1u8 << unused_bits) - 1) == 0)
}

async fn remove_stale(path: &Path) {
    if let Err(error) = tokio::fs::remove_file(path).await
        && error.kind() != std::io::ErrorKind::NotFound
    {
        tracing::debug!(path = %path.display(), %error, "Failed to remove stale BitTorrent checkpoint");
    }
}

#[cfg(test)]
mod tests {
    use super::BtCheckpoint;
    use crate::filesystem::control_file::{ControlFile, ControlFileInFlightPiece};

    #[tokio::test]
    async fn checkpoint_restores_piece_sized_completed_length() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 10]).unwrap();
        let info_hash = [0x11; 20];
        let mut checkpoint = BtCheckpoint::open(&output, true, 10, 4, 3, info_hash)
            .await
            .unwrap();

        checkpoint.save(&[0b1010_0000], 6).await.unwrap();
        assert_eq!(checkpoint.completed_length(), 6);

        let restored = BtCheckpoint::open(&output, true, 10, 4, 3, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0b1010_0000].as_slice()));
        assert_eq!(restored.completed_length(), 6);
        let control = ControlFile::load(&ControlFile::control_path_for(&output))
            .await
            .unwrap()
            .unwrap();
        assert_eq!(control.torrent_piece_length(), Some(4));
    }

    #[tokio::test]
    async fn checkpoint_restores_only_durable_in_flight_blocks() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, vec![0xA5; 64 * 1024]).unwrap();
        let info_hash = [0x29; 20];
        let mut checkpoint = BtCheckpoint::open(&output, true, 64 * 1024, 64 * 1024, 1, info_hash)
            .await
            .unwrap();
        let partial = ControlFileInFlightPiece {
            index: 0,
            length: 64 * 1024,
            bitfield: vec![0b1010_0000],
        };

        checkpoint
            .save_with_in_flight_pieces(&[0], 0, std::slice::from_ref(&partial))
            .await
            .unwrap();

        let restored = BtCheckpoint::open(&output, true, 64 * 1024, 64 * 1024, 1, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0].as_slice()));
        assert_eq!(restored.in_flight_pieces(), &[partial]);
    }

    #[tokio::test]
    async fn checkpoint_accepts_a_multi_file_output_directory() {
        let dir = tempfile::tempdir().unwrap();
        let info_hash = [0x77; 20];
        let mut checkpoint = BtCheckpoint::open(dir.path(), false, 8, 4, 2, info_hash)
            .await
            .expect("a multi-file output directory needs a sidecar checkpoint");

        checkpoint.save(&[0xC0], 8).await.unwrap();

        let sidecar = ControlFile::control_path_for(dir.path());
        assert!(sidecar.is_file());
        let restored = BtCheckpoint::open(dir.path(), true, 8, 4, 2, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.completed_length(), 8);
    }

    #[tokio::test]
    async fn checkpoint_discards_different_piece_length() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 10]).unwrap();
        let info_hash = [0x55; 20];
        let mut checkpoint = BtCheckpoint::open(&output, true, 10, 4, 3, info_hash)
            .await
            .unwrap();
        checkpoint.save(&[0b1100_0000], 8).await.unwrap();

        let restored = BtCheckpoint::open(&output, true, 10, 5, 3, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0u8].as_slice()));
        assert_eq!(restored.completed_length(), 0);
    }

    #[tokio::test]
    async fn checkpoint_accepts_legacy_sidecar_without_piece_length() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 8]).unwrap();
        let info_hash = [0x66; 20];
        let path = ControlFile::control_path_for(&output);

        // Retired A2CF bytes: torrent marker + info hash, but no persisted
        // piece length. The next save must rewrite this as native v1.
        let mut legacy = Vec::new();
        legacy.extend_from_slice(b"A2CF");
        legacy.extend_from_slice(&1u16.to_le_bytes());
        legacy.push(0x02 | 0x04);
        legacy.extend_from_slice(&8u64.to_le_bytes());
        legacy.extend_from_slice(&8u64.to_le_bytes());
        legacy.extend_from_slice(&0u64.to_le_bytes());
        legacy.extend_from_slice(&1u64.to_le_bytes());
        legacy.extend_from_slice(&info_hash);
        legacy.push(0xC0);
        tokio::fs::write(&path, legacy).await.unwrap();

        let mut restored = BtCheckpoint::open(&output, true, 8, 4, 2, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0xC0].as_slice()));
        restored.save(&[0xC0], 8).await.unwrap();

        let upgraded = ControlFile::load(&path).await.unwrap().unwrap();
        assert_eq!(upgraded.torrent_piece_length(), Some(4));
    }

    #[tokio::test]
    async fn checkpoint_discards_different_torrent_identity() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 8]).unwrap();
        let first_hash = [0x11; 20];
        let second_hash = [0x22; 20];
        let mut checkpoint = BtCheckpoint::open(&output, true, 8, 4, 2, first_hash)
            .await
            .unwrap();
        checkpoint.save(&[0xC0], 4).await.unwrap();

        let mut restored = BtCheckpoint::open(&output, true, 8, 4, 2, second_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0u8].as_slice()));
        assert_eq!(restored.completed_length(), 0);
        restored.save(&[0u8], 0).await.unwrap();
        let control = ControlFile::load(&ControlFile::control_path_for(&output))
            .await
            .unwrap()
            .unwrap();
        assert_eq!(control.torrent_info_hash(), Some(second_hash));
    }

    #[tokio::test]
    async fn checkpoint_rejects_set_trailing_bits() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 10]).unwrap();
        let info_hash = [0x33; 20];
        let mut checkpoint = BtCheckpoint::open(&output, true, 10, 4, 3, info_hash)
            .await
            .unwrap();

        assert!(checkpoint.save(&[0b0010_0001], 4).await.is_err());
        assert_eq!(checkpoint.bitfield(), Some([0u8].as_slice()));
    }

    #[tokio::test]
    async fn checkpoint_discards_sidecar_without_payload() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        let info_hash = [0x44; 20];
        let mut checkpoint = BtCheckpoint::open(&output, false, 8, 4, 2, info_hash)
            .await
            .unwrap();
        checkpoint.save(&[0xC0], 4).await.unwrap();

        let restored = BtCheckpoint::open(&output, false, 8, 4, 2, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0u8].as_slice()));
        assert_eq!(restored.completed_length(), 0);
    }

    #[tokio::test]
    async fn checkpoint_ignores_stale_temporary_sidecar() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 8]).unwrap();
        let info_hash = [0x77; 20];
        let path = ControlFile::control_path_for(&output);

        let mut checkpoint = BtCheckpoint::open(&output, true, 8, 4, 2, info_hash)
            .await
            .unwrap();
        checkpoint.save(&[0x80], 4).await.unwrap();
        tokio::fs::write(path.with_extension("aria2.tmp"), b"truncated")
            .await
            .unwrap();

        let restored = BtCheckpoint::open(&output, true, 8, 4, 2, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0x80].as_slice()));
        assert!(path.with_extension("aria2.tmp").exists());
    }

    #[tokio::test]
    async fn checkpoint_discards_truncated_sidecar() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 8]).unwrap();
        let info_hash = [0x88; 20];
        let path = ControlFile::control_path_for(&output);
        tokio::fs::write(&path, b"A2CF\x01\x00").await.unwrap();

        let restored = BtCheckpoint::open(&output, true, 8, 4, 2, info_hash)
            .await
            .unwrap();
        assert_eq!(restored.bitfield(), Some([0u8].as_slice()));
        assert!(
            !path.exists(),
            "stale truncated checkpoint should be removed"
        );
    }

    #[tokio::test]
    async fn checkpoint_save_failure_does_not_advance_memory() {
        let dir = tempfile::tempdir().unwrap();
        let output = dir.path().join("payload.bin");
        std::fs::write(&output, [0u8; 8]).unwrap();
        let info_hash = [0x99; 20];
        let path = ControlFile::control_path_for(&output);
        let mut checkpoint = BtCheckpoint::open(&output, true, 8, 4, 2, info_hash)
            .await
            .unwrap();
        checkpoint.save(&[0x80], 4).await.unwrap();

        std::fs::remove_file(&output).unwrap();
        std::fs::remove_file(&path).unwrap();
        std::fs::remove_dir(dir.path()).unwrap();
        assert!(checkpoint.save(&[0xC0], 8).await.is_err());
        assert_eq!(checkpoint.bitfield(), Some([0x80].as_slice()));
        assert_eq!(checkpoint.completed_length(), 4);
    }
}