webfetch_core/
http.rs

1//! HTTP primitives shared by the fetch and search paths: the user agent, the
2//! response-body cap, and the retry classification. Both paths previously
3//! carried their own copy of the last two and had drifted apart — search read
4//! bodies without any cap at all.
5
6use reqwest::{Response, StatusCode};
7
8/// The user agent both paths send.
9///
10/// A tool-shaped agent (`webfetch/0.1.x`) is refused or challenged by a large
11/// share of CDNs, which turns an ordinary fetch into an empty page. The search
12/// path already sent a browser agent for exactly this reason; the fetch path
13/// did not, and the mismatch was the single most common cause of a 403.
14pub const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
15
16/// Hard cap on the response body we will read (5 MiB). The HTML extractor turns
17/// a page into a few KB of text, so a multi-megabyte body is almost never worth
18/// the bandwidth, memory, and parse time — and an unbounded read is a DoS lever.
19/// Bodies over the cap are *truncated* (not errored): partial content is still
20/// useful and the extractor copes with truncated HTML.
21///
22/// The cap counts bytes *after* transparent gzip/brotli decoding, so it also
23/// bounds a decompression bomb.
24pub const MAX_BODY_BYTES: usize = 5 * 1024 * 1024;
25
26/// Append as much of `chunk` to `buf` as fits under `max`. Returns `true` once
27/// the cap is reached (the body is truncated and the caller should stop).
28fn push_capped(buf: &mut Vec<u8>, chunk: &[u8], max: usize) -> bool {
29    let remaining = max.saturating_sub(buf.len());
30    if chunk.len() >= remaining {
31        buf.extend_from_slice(&chunk[..remaining]);
32        true
33    } else {
34        buf.extend_from_slice(chunk);
35        false
36    }
37}
38
39/// Read a response body, streaming chunks with a running byte cap so an
40/// oversized body is bounded before it is ever parsed. The `bool` in the error
41/// reports whether the read failure is transient (worth retrying).
42///
43/// Decodes as UTF-8. Use [`read_body_capped_bytes`] when the response's declared
44/// charset matters.
45pub async fn read_body_capped(resp: Response) -> Result<String, (anyhow::Error, bool)> {
46    let bytes = read_body_capped_bytes(resp).await?;
47    Ok(String::from_utf8_lossy(&bytes).into_owned())
48}
49
50/// [`read_body_capped`] without the decoding step, for callers that need to
51/// apply the response's declared charset themselves.
52pub async fn read_body_capped_bytes(mut resp: Response) -> Result<Vec<u8>, (anyhow::Error, bool)> {
53    let mut buf: Vec<u8> = Vec::new();
54    // Honour Content-Length to pre-size, but never trust it past the cap.
55    if let Some(len) = resp.content_length() {
56        buf.reserve(len.min(MAX_BODY_BYTES as u64) as usize);
57    }
58    loop {
59        match resp.chunk().await {
60            Ok(Some(chunk)) => {
61                if push_capped(&mut buf, &chunk, MAX_BODY_BYTES) {
62                    break;
63                }
64            }
65            Ok(None) => break,
66            Err(e) => {
67                let transient = e.is_timeout();
68                return Err((e.into(), transient));
69            }
70        }
71    }
72    Ok(buf)
73}
74
75/// Is a send/connect failure worth retrying?
76pub fn transient_send_error(e: &reqwest::Error) -> bool {
77    e.is_timeout() || e.is_connect() || e.is_request()
78}
79
80/// Is a response status worth retrying? Server errors and explicit throttling.
81pub fn transient_status(status: StatusCode) -> bool {
82    status.is_server_error() || status.as_u16() == 429
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88
89    #[test]
90    fn push_capped_truncates_oversized_chunk() {
91        let mut buf = Vec::new();
92        // A single chunk larger than the cap is clipped to the cap.
93        let stopped = push_capped(&mut buf, &[b'x'; 10], 4);
94        assert!(stopped);
95        assert_eq!(buf.len(), 4);
96    }
97
98    #[test]
99    fn push_capped_accumulates_until_cap() {
100        let mut buf = Vec::new();
101        assert!(!push_capped(&mut buf, b"abc", 8));
102        assert!(!push_capped(&mut buf, b"de", 8));
103        assert_eq!(buf, b"abcde");
104        // Next chunk crosses the cap: only the remaining 3 bytes are kept.
105        let stopped = push_capped(&mut buf, b"fghij", 8);
106        assert!(stopped);
107        assert_eq!(buf.len(), 8);
108        assert_eq!(buf, b"abcdefgh");
109    }
110
111    #[test]
112    fn push_capped_small_body_unaffected() {
113        let mut buf = Vec::new();
114        let stopped = push_capped(&mut buf, b"hello", 1024);
115        assert!(!stopped);
116        assert_eq!(buf, b"hello");
117    }
118
119    #[test]
120    fn transient_status_covers_5xx_and_429() {
121        assert!(transient_status(StatusCode::INTERNAL_SERVER_ERROR));
122        assert!(transient_status(StatusCode::TOO_MANY_REQUESTS));
123        assert!(!transient_status(StatusCode::NOT_FOUND));
124        assert!(!transient_status(StatusCode::FORBIDDEN));
125    }
126}
webfetch_core/http.rs

webfetch_core/
http.rs
