Skip to main content

webfetch_core/
http.rs

1//! HTTP primitives shared by the fetch and search paths: the user agent, the
2//! response-body cap, and the retry classification. Both paths previously
3//! carried their own copy of the last two and had drifted apart — search read
4//! bodies without any cap at all.
5//!
6//! Nothing here touches a concrete HTTP client; the transport-level pieces live
7//! in [`crate::transport`].
8
9/// The user agent both paths send.
10///
11/// A tool-shaped agent (`webfetch/0.1.x`) is refused or challenged by a large
12/// share of CDNs, which turns an ordinary fetch into an empty page. The search
13/// path already sent a browser agent for exactly this reason; the fetch path
14/// did not, and the mismatch was the single most common cause of a 403.
15pub const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";
16
17/// Hard cap on the response body we will read (5 MiB). The HTML extractor turns
18/// a page into a few KB of text, so a multi-megabyte body is almost never worth
19/// the bandwidth, memory, and parse time — and an unbounded read is a DoS lever.
20/// Bodies over the cap are *truncated* (not errored): partial content is still
21/// useful and the extractor copes with truncated HTML.
22///
23/// The cap counts bytes *after* transparent gzip/brotli decoding, so it also
24/// bounds a decompression bomb.
25pub const MAX_BODY_BYTES: usize = 5 * 1024 * 1024;
26
27/// Cap on a 4xx or 5xx body (64 KiB). Error bodies only feed a message, and
28/// the provider's own explanation fits well inside this.
29pub const ERROR_BODY_CAP: usize = 64 * 1024;
30
31/// Append as much of `chunk` to `buf` as fits under `max`. Returns `true` once
32/// the cap is reached (the body is truncated and the caller should stop).
33#[cfg_attr(not(any(feature = "native", feature = "wasi")), allow(dead_code))]
34pub(crate) fn push_capped(buf: &mut Vec<u8>, chunk: &[u8], max: usize) -> bool {
35    let remaining = max.saturating_sub(buf.len());
36    if chunk.len() >= remaining {
37        buf.extend_from_slice(&chunk[..remaining]);
38        true
39    } else {
40        buf.extend_from_slice(chunk);
41        false
42    }
43}
44
45/// Is a response status worth retrying? Server errors and explicit throttling.
46pub fn transient_status(status: u16) -> bool {
47    (500..600).contains(&status) || status == 429
48}
49
50/// A status code and its reason phrase, as `404 Not Found`. A code with no
51/// standard phrase is shown as the number alone.
52///
53/// Native builds take the phrase from reqwest's `StatusCode`, so every
54/// registered code is covered. The WASI build has no HTTP crate of its own and
55/// uses a table of the codes the libraries meet in practice.
56pub fn status_text(status: u16) -> String {
57    #[cfg(feature = "native")]
58    let reason = reqwest::StatusCode::from_u16(status)
59        .ok()
60        .and_then(|code| code.canonical_reason());
61    #[cfg(not(feature = "native"))]
62    let reason = wasi_reason(status);
63    match reason {
64        Some(reason) => format!("{status} {reason}"),
65        None => status.to_string(),
66    }
67}
68
69#[cfg(not(feature = "native"))]
70fn wasi_reason(status: u16) -> Option<&'static str> {
71    Some(match status {
72        200 => "OK",
73        204 => "No Content",
74        301 => "Moved Permanently",
75        302 => "Found",
76        303 => "See Other",
77        304 => "Not Modified",
78        307 => "Temporary Redirect",
79        308 => "Permanent Redirect",
80        400 => "Bad Request",
81        401 => "Unauthorized",
82        403 => "Forbidden",
83        404 => "Not Found",
84        405 => "Method Not Allowed",
85        408 => "Request Timeout",
86        409 => "Conflict",
87        410 => "Gone",
88        413 => "Payload Too Large",
89        415 => "Unsupported Media Type",
90        422 => "Unprocessable Entity",
91        429 => "Too Many Requests",
92        500 => "Internal Server Error",
93        501 => "Not Implemented",
94        502 => "Bad Gateway",
95        503 => "Service Unavailable",
96        504 => "Gateway Timeout",
97        _ => return None,
98    })
99}
100
101#[cfg(test)]
102mod tests {
103    use super::*;
104
105    #[test]
106    fn status_text_names_the_reason_phrase() {
107        assert_eq!(status_text(404), "404 Not Found");
108        assert_eq!(status_text(401), "401 Unauthorized");
109        // No standard phrase: the number alone.
110        assert_eq!(status_text(799), "799");
111    }
112
113    #[test]
114    fn push_capped_truncates_oversized_chunk() {
115        let mut buf = Vec::new();
116        // A single chunk larger than the cap is clipped to the cap.
117        let stopped = push_capped(&mut buf, &[b'x'; 10], 4);
118        assert!(stopped);
119        assert_eq!(buf.len(), 4);
120    }
121
122    #[test]
123    fn push_capped_accumulates_until_cap() {
124        let mut buf = Vec::new();
125        assert!(!push_capped(&mut buf, b"abc", 8));
126        assert!(!push_capped(&mut buf, b"de", 8));
127        assert_eq!(buf, b"abcde");
128        // Next chunk crosses the cap: only the remaining 3 bytes are kept.
129        let stopped = push_capped(&mut buf, b"fghij", 8);
130        assert!(stopped);
131        assert_eq!(buf.len(), 8);
132        assert_eq!(buf, b"abcdefgh");
133    }
134
135    #[test]
136    fn push_capped_small_body_unaffected() {
137        let mut buf = Vec::new();
138        let stopped = push_capped(&mut buf, b"hello", 1024);
139        assert!(!stopped);
140        assert_eq!(buf, b"hello");
141    }
142
143    #[test]
144    fn transient_status_covers_5xx_and_429() {
145        assert!(transient_status(500));
146        assert!(transient_status(503));
147        assert!(transient_status(429));
148        assert!(!transient_status(404));
149        assert!(!transient_status(403));
150        assert!(!transient_status(202));
151    }
152}