downlowd 0.1.1

Download files with automatic retries and resumes.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

use std::{
    collections::HashMap, fs, io::Write, path::{Path, PathBuf}, sync::{Mutex, OnceLock}
};

use rand::Fill;

mod progress_recorder;
pub use progress_recorder::*;

pub struct HeadData {
    pub last_modified: String,
    pub etag: String,
    pub content_length: u64,
}

#[derive(Clone)]
pub struct BigFile {
    pub path: PathBuf,
    pub url: String,
}

pub fn head_url(url: &str) -> HeadData {
    let response = ureq::head(url).call().expect("url to exist");

    assert_eq!(response.status(), 200, "Resource {url} should exist");

    let last_modified = response
        .headers()
        .get(http::header::LAST_MODIFIED)
        .map(|s| s.to_str().expect("valid string").to_owned())
        .unwrap();

    let etag = response
        .headers()
        .get(http::header::ETAG)
        .map(|s| s.to_str().expect("valid string").to_owned())
        .unwrap();

    let content_length = response
        .headers()
        .get(http::header::CONTENT_LENGTH)
        .map(|s| {
            s.to_str()
                .expect("valid string")
                .parse::<u64>()
                .expect("valid u64")
        })
        .unwrap();

    HeadData {
        last_modified,
        etag,
        content_length,
    }
}

static FILES: OnceLock<Mutex<HashMap<usize, BigFile>>> = OnceLock::new();

fn big_file(size: usize) -> BigFile {
    let file_map = FILES.get_or_init(|| Mutex::new(HashMap::new()));

    let mut map = file_map.lock().unwrap();
    if let Some(bf) = map.get(&size) {
        return bf.clone();
    }

    let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .join("test-support")
        .join("static")
        .join(format!("big-file-{size}.bin"));

    // Check if the file already exists and is the correct size.
    let file_size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0) as usize;
    if file_size != size {
        let mut file = std::fs::File::create(&path).expect("create file");

        // Write random data to the file.
        let mut remaining = size;
        let mut buffer = [0u8; 4096];
        let mut rng = rand::rng();
        while remaining > 0 {
            buffer.fill(&mut rng);
            let buffer = &buffer[..remaining.min(buffer.len())];
            file.write_all(buffer).expect("write to file");
            remaining -= buffer.len();
        }
    }

    let result = BigFile {
        path,
        url: format!("/big-file-{size}.bin"),
    };
    map.insert(size, result.clone());
    result
}

/// Returns the URL path to a file of a specific size for testing purposes.
/// The file is created on first use in the `test-support/static` directory.
pub fn big_file_url(size: usize) -> String {
    big_file(size).url
}

/// Return the path on the file system to a big file of the specified size.
/// This will be the same file as returned by `big_file_url`.
pub fn big_file_path(size: usize) -> PathBuf {
    big_file(size).path
}

pub fn write_sidecar_file(
    path: &Path,
    last_modified: Option<&str>,
    etag: Option<&str>,
    content_length: Option<u64>,
) -> std::io::Result<()> {
    let mut contents = String::new();
    if let Some(last_modified) = last_modified {
        contents.push_str(&format!("Last-Modified: {last_modified}\n"));
    }
    if let Some(etag) = etag {
        contents.push_str(&format!("Etag: {etag}\n",));
    }
    if let Some(content_length) = content_length {
        contents.push_str(&format!("File-Length: {content_length}\n"));
    }
    fs::write(path, contents)?;
    Ok(())
}