bytehaul 0.2.0

Async HTTP download library with resume, multi-connection, rate limiting, and checksum verification
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157

use std::time::{Duration, Instant};

use bytehaul::{Checksum, DownloadSpec, Downloader, FileAllocation};
use sha2::{Digest, Sha256};
use warp::Filter;

fn hex_encode(bytes: &[u8]) -> String {
    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}

#[tokio::test]
async fn test_rate_limiting() {
    // 10 KB of data
    let data = vec![0xABu8; 10 * 1024];
    let expected = data.clone();

    let route = warp::path("file").map(move || {
        warp::http::Response::builder()
            .header("content-length", data.len().to_string())
            .body(data.clone())
            .unwrap()
    });
    let (addr, server) = warp::serve(route).bind_ephemeral(([127, 0, 0, 1], 0));
    tokio::spawn(server);

    let dir = tempfile::tempdir().unwrap();
    let out = dir.path().join("rate_limited.bin");

    // Limit to ~5 KB/s — should take at least 1.5s for 10 KB
    let spec = DownloadSpec::new(format!("http://{addr}/file"))
        .output_path(out.clone())
        .max_connections(1)
        .file_allocation(FileAllocation::None)
        .resume(false)
        .max_download_speed(5 * 1024);

    let dl = Downloader::builder().build().unwrap();
    let start = Instant::now();
    let handle = dl.download(spec);
    handle.wait().await.unwrap();
    let elapsed = start.elapsed();

    // With 10KB at 5KB/s, should take ~2 seconds. Allow at least 1s.
    assert!(
        elapsed >= Duration::from_secs(1),
        "rate limiting should slow the download, but it finished in {elapsed:?}"
    );

    let content = std::fs::read(&out).unwrap();
    assert_eq!(content, expected);
}

#[tokio::test]
async fn test_no_rate_limit_is_fast() {
    let data = vec![0xCDu8; 100 * 1024];
    let expected = data.clone();

    let route = warp::path("file").map(move || warp::http::Response::new(data.clone()));
    let (addr, server) = warp::serve(route).bind_ephemeral(([127, 0, 0, 1], 0));
    tokio::spawn(server);

    let dir = tempfile::tempdir().unwrap();
    let out = dir.path().join("no_limit.bin");

    let spec = DownloadSpec::new(format!("http://{addr}/file"))
        .output_path(out.clone())
        .max_connections(1)
        .file_allocation(FileAllocation::None)
        .resume(false)
        .max_download_speed(0);

    let dl = Downloader::builder().build().unwrap();
    let start = Instant::now();
    let handle = dl.download(spec);
    handle.wait().await.unwrap();
    let elapsed = start.elapsed();

    assert!(
        elapsed < Duration::from_secs(5),
        "unlimited download should be fast, but took {elapsed:?}"
    );

    let content = std::fs::read(&out).unwrap();
    assert_eq!(content, expected);
}

#[tokio::test]
async fn test_checksum_sha256_pass() {
    let body = b"Hello, checksum world!";
    let hash = Sha256::digest(body);
    let hex = hex_encode(hash.as_slice());

    let data = body.to_vec();
    let route = warp::path("file").map(move || warp::http::Response::new(data.clone()));
    let (addr, server) = warp::serve(route).bind_ephemeral(([127, 0, 0, 1], 0));
    tokio::spawn(server);

    let dir = tempfile::tempdir().unwrap();
    let out = dir.path().join("checksum_ok.bin");

    let spec = DownloadSpec::new(format!("http://{addr}/file"))
        .output_path(out.clone())
        .max_connections(1)
        .file_allocation(FileAllocation::None)
        .resume(false)
        .checksum(Checksum::Sha256(hex));

    let dl = Downloader::builder().build().unwrap();
    let handle = dl.download(spec);
    handle.wait().await.unwrap();

    let content = std::fs::read(&out).unwrap();
    assert_eq!(content, body.as_slice());
}

#[tokio::test]
async fn test_checksum_sha256_fail() {
    let body = b"Hello, checksum world!";
    let data = body.to_vec();

    let route = warp::path("file").map(move || warp::http::Response::new(data.clone()));
    let (addr, server) = warp::serve(route).bind_ephemeral(([127, 0, 0, 1], 0));
    tokio::spawn(server);

    let dir = tempfile::tempdir().unwrap();
    let out = dir.path().join("checksum_bad.bin");

    let spec = DownloadSpec::new(format!("http://{addr}/file"))
        .output_path(out.clone())
        .max_connections(1)
        .file_allocation(FileAllocation::None)
        .resume(false)
        .checksum(Checksum::Sha256(
            "0000000000000000000000000000000000000000000000000000000000000000".to_string(),
        ));

    let dl = Downloader::builder().build().unwrap();
    let handle = dl.download(spec);
    let result = handle.wait().await;
    assert!(result.is_err(), "should fail with checksum mismatch");
    let err_str = result.unwrap_err().to_string();
    assert!(
        err_str.contains("checksum mismatch"),
        "error should mention checksum mismatch, got: {err_str}"
    );
}

#[tokio::test]
async fn test_checksum_error_is_public() {
    let err = bytehaul::DownloadError::ChecksumMismatch {
        expected: "abc".into(),
        actual: "def".into(),
    };
    assert!(err.to_string().contains("abc"));
    assert!(err.to_string().contains("def"));
    assert!(!err.is_retryable());
}