fast_down/http/
prefetch.rs

1//! Prefetch metadata for a downloadable URL over HTTP.
2//!
3//! [`Prefetch::prefetch`] issues the initial GET (and a range probe) through a
4//! [`crate::http::HttpClient`], then assembles a [`crate::UrlInfo`] describing
5//! the resource: its size, suggested filename, content type, range support, and
6//! the [`crate::FileId`] used for resumable downloads.
7
8use crate::{
9    UrlInfo,
10    http::{
11        ContentDisposition, GetRequestError, GetResponse, HttpClient, HttpError, HttpHeaders,
12        HttpRequestBuilder, HttpResponse,
13    },
14    url_info::FileId,
15};
16use fast_pull::PullResult;
17use std::{borrow::Borrow, future::Future};
18use url::Url;
19
20/// Result of a prefetch operation: the metadata ([`UrlInfo`]) and the initial HTTP response.
21pub type PrefetchResult<Client> =
22    PullResult<(UrlInfo, GetResponse<Client>), GetRequestError<Client>>;
23
24/// Trait for fetching resource metadata (size, filename, range support) from a URL.
25///
26/// Implementors perform a GET request to gather [`UrlInfo`] and
27/// the initial response for subsequent downloading.
28pub trait Prefetch<Client: HttpClient> {
29    /// Fetch the resource metadata and the initial response for `url`.
30    fn prefetch(&self, url: Url) -> impl Future<Output = PrefetchResult<Client>> + Send;
31}
32
33impl<Client, BorrowClient> Prefetch<Client> for BorrowClient
34where
35    Client: HttpClient,
36    BorrowClient: Borrow<Client> + Sync,
37{
38    async fn prefetch(&self, url: Url) -> PrefetchResult<Client> {
39        prefetch(self.borrow(), url).await
40    }
41}
42
43fn get_filename(headers: &impl HttpHeaders, url: &Url) -> String {
44    headers
45        .get("content-disposition")
46        .ok()
47        .and_then(|s| ContentDisposition::parse(s.as_ref()).filename)
48        .filter(|s| !s.trim().is_empty())
49        .or_else(|| {
50            url.path_segments()
51                .and_then(|mut segments| segments.next_back())
52                .map(|s| {
53                    let s = urlencoding::decode_binary(s.as_bytes());
54                    String::from_utf8_lossy(&s).into_owned()
55                })
56                .filter(|s| !s.trim().is_empty())
57        })
58        .or_else(|| url.host_str().map(|s| s.replace('.', "_")))
59        .unwrap_or_else(|| url.to_string().replace('.', "_"))
60}
61
62async fn prefetch<Client: HttpClient>(client: &Client, url: Url) -> PrefetchResult<Client> {
63    let (no_range_fut, range_fut) = (
64        prefetch_no_range(client, url.clone()),
65        is_support_range(client, url),
66    );
67    let (result_no_range, result_range) = tokio::join!(no_range_fut, range_fut);
68    let mut res = result_no_range?;
69    if matches!(result_range, Ok(true)) {
70        res.0.supports_range = true;
71        if res.0.size != 0 {
72            res.0.fast_download = true;
73        }
74    }
75    Ok(res)
76}
77
78async fn prefetch_no_range<Client: HttpClient>(
79    client: &Client,
80    url: Url,
81) -> PrefetchResult<Client> {
82    let resp = client.get(url, None).send().await?;
83    let headers = resp.headers();
84    let size = headers
85        .get("content-length")
86        .ok()
87        .and_then(|v| v.parse().ok())
88        .unwrap_or(0);
89    let final_url = resp.url();
90    Ok((
91        UrlInfo {
92            final_url: final_url.clone(),
93            raw_name: get_filename(headers, final_url),
94            size,
95            supports_range: false,
96            fast_download: false,
97            file_id: FileId::new(
98                headers.get("etag").ok().as_deref(),
99                headers.get("last-modified").ok().as_deref(),
100            ),
101            content_type: headers.get("content-type").ok().map(String::from),
102        },
103        resp,
104    ))
105}
106
107async fn is_support_range<Client: HttpClient>(
108    client: &Client,
109    url: Url,
110) -> PullResult<bool, HttpError<Client>> {
111    let resp = client
112        .get(url, Some(0..1))
113        .send()
114        .await
115        .map_err(|(e, d)| (HttpError::Request(e), d))?;
116    let headers = resp.headers();
117    let supports_range = headers
118        .get("content-range")
119        .is_ok_and(|v| v.trim_start().to_lowercase().starts_with("bytes 0-0/"));
120    Ok(supports_range)
121}
122
123#[cfg(test)]
124mod tests {
125    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
126    use super::*;
127    use std::borrow::Cow;
128    use std::collections::HashMap;
129
130    struct MapHeaders(HashMap<String, String>);
131    impl HttpHeaders for MapHeaders {
132        type GetHeaderError = std::convert::Infallible;
133        fn get(&self, header: &str) -> Result<Cow<'_, str>, Self::GetHeaderError> {
134            Ok(Cow::Borrowed(self.0.get(header).map_or("", String::as_str)))
135        }
136    }
137
138    #[test]
139    fn get_filename_filename_star_not_double_decoded() {
140        // Hypothesis A: `ContentDisposition::parse` already percent-decodes
141        // `filename*` ("%25100" -> "%100"). `get_filename` then calls
142        // `urlencoding::decode_binary` on the decoded string, double-decoding
143        // it. A server that intends the literal filename "%100" (sent as
144        // `filename*=UTF-8''%25100`) must survive unchanged.
145        let mut h = HashMap::new();
146        h.insert(
147            "content-disposition".to_string(),
148            "attachment; filename*=UTF-8''%25100".to_string(),
149        );
150        let headers = MapHeaders(h);
151        let url = Url::parse("http://example.com/").unwrap();
152        assert_eq!(get_filename(&headers, &url), "%100");
153    }
154
155    #[test]
156    fn get_filename_filename_star_no_percent_passthrough() {
157        // Control: when the decoded `filename*` contains no literal '%' byte,
158        // `decode_binary` leaves it untouched, so a normal UTF-8 name is correct.
159        let mut h = HashMap::new();
160        h.insert(
161            "content-disposition".to_string(),
162            "attachment; filename*=UTF-8''%E6%B5%8B%E8%AF%95.txt".to_string(),
163        );
164        let headers = MapHeaders(h);
165        let url = Url::parse("http://example.com/").unwrap();
166        assert_eq!(get_filename(&headers, &url), "测试.txt");
167    }
168
169    #[test]
170    fn get_filename_plain_filename_percent_stays_literal() {
171        // Plain `filename` is no longer percent-decoded (RFC 6266: quoted-string
172        // is literal). "%E6%B5%8B.txt" stays as-is, unlike the old behavior.
173        let mut h = HashMap::new();
174        h.insert(
175            "content-disposition".to_string(),
176            "attachment; filename=\"%E6%B5%8B.txt\"".to_string(),
177        );
178        let headers = MapHeaders(h);
179        let url = Url::parse("http://example.com/").unwrap();
180        assert_eq!(get_filename(&headers, &url), "%E6%B5%8B.txt");
181    }
182
183    #[test]
184    fn get_filename_url_path_fallback_decodes_percent() {
185        // No content-disposition → falls back to the last URL path segment,
186        // which IS percent-decoded (URL paths are always encoded).
187        let headers = MapHeaders(HashMap::new());
188        let url = Url::parse("http://example.com/dir/%E6%B5%8B%E8%AF%95.txt").unwrap();
189        assert_eq!(get_filename(&headers, &url), "测试.txt");
190    }
191
192    #[test]
193    fn get_filename_url_path_invalid_utf8_uses_replacement() {
194        // %FF is not valid UTF-8; from_utf8_lossy produces U+FFFD.
195        let headers = MapHeaders(HashMap::new());
196        let url = Url::parse("http://example.com/%FF.txt").unwrap();
197        assert_eq!(get_filename(&headers, &url), "\u{FFFD}.txt");
198    }
199
200    #[test]
201    fn get_filename_whitespace_cd_falls_through_to_url() {
202        // Whitespace-only filename from CD is filtered out → URL path is used.
203        let mut h = HashMap::new();
204        h.insert(
205            "content-disposition".to_string(),
206            "attachment; filename=\"   \"".to_string(),
207        );
208        let headers = MapHeaders(h);
209        let url = Url::parse("http://example.com/report.pdf").unwrap();
210        assert_eq!(get_filename(&headers, &url), "report.pdf");
211    }
212
213    #[test]
214    fn get_filename_empty_path_falls_back_to_host() {
215        // Root URL (empty path segment) → host with dots replaced by underscores.
216        let headers = MapHeaders(HashMap::new());
217        let url = Url::parse("http://cdn.example.com/").unwrap();
218        assert_eq!(get_filename(&headers, &url), "cdn_example_com");
219    }
220
221    #[test]
222    fn get_filename_path_segment_spaces_fall_back_to_host() {
223        // A path segment that is only spaces decodes to whitespace → filtered →
224        // host fallback.
225        let headers = MapHeaders(HashMap::new());
226        let url = Url::parse("http://cdn.example.com/%20%20").unwrap();
227        assert_eq!(get_filename(&headers, &url), "cdn_example_com");
228    }
229}
fast_down/http/prefetch.rs

fast_down/http/
prefetch.rs
