webfetch/
fetch.rs

1use reqwest::header::{CONTENT_TYPE, LOCATION};
2use reqwest::{redirect::Policy, Client};
3use std::net::SocketAddr;
4use std::time::{Duration, Instant};
5
6use crate::guard;
7use crate::tls::TlsConfig;
8use webfetch_core::charset;
9use webfetch_core::http::{
10    read_body_capped_bytes, transient_send_error, transient_status, USER_AGENT,
11};
12
13const MAX_ATTEMPTS: u32 = 3;
14const MAX_REDIRECTS: usize = 5;
15
16/// Multiplier turning the per-request `--timeout` into a budget for the whole
17/// fetch.
18///
19/// `--timeout` bounds one request. With retries and redirects a single fetch
20/// could issue `MAX_ATTEMPTS * (MAX_REDIRECTS + 1)` requests, so `--timeout 10`
21/// could keep running for minutes — not what anyone setting a timeout expects.
22/// The whole fetch now shares one deadline, and each request gets whatever is
23/// left of it.
24const TOTAL_BUDGET_MULTIPLIER: u32 = 3;
25
26/// Outcome of an HTTP fetch: the body, the URL we actually landed on after
27/// following redirects, and the response's `Content-Type` (if any).
28#[derive(Debug, Clone)]
29pub struct FetchedPage {
30    pub body: String,
31    pub final_url: String,
32    pub content_type: Option<String>,
33    /// Set when the page declared a charset this build cannot decode, so the
34    /// body was read as UTF-8 and may be garbled.
35    pub undecodable_charset: Option<String>,
36}
37
38/// Find `<meta charset=…>` in the head of a body whose header declared nothing.
39///
40/// Only the first 2 KiB are searched: the declaration is required to appear
41/// early, and scanning a whole 5 MiB body for it would be wasted work.
42fn sniff_meta_charset(raw: &[u8]) -> Option<String> {
43    const WINDOW: usize = 2048;
44    let head = &raw[..raw.len().min(WINDOW)];
45    let text = String::from_utf8_lossy(head).to_ascii_lowercase();
46    let at = text.find("charset")? + "charset".len();
47    let rest = text[at..].trim_start().strip_prefix('=')?.trim_start();
48    let value: String = rest
49        .trim_start_matches(['"', '\''])
50        .chars()
51        .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
52        .collect();
53    (!value.is_empty()).then_some(value)
54}
55
56/// One hop's result: either the final page, or a redirect to a raw `Location`.
57enum Hop {
58    Page(FetchedPage),
59    Redirect(String),
60}
61
62/// Build a client for a single validated URL. `pinned` are the public IPs the
63/// host already resolved to; binding them closes the DNS-rebinding window
64/// between validation and connection.
65///
66/// Redirects are **not** followed by reqwest here ([`Policy::none`]): we follow
67/// them manually in [`fetch_page`] so every hop is re-validated *and* pinned to
68/// its own resolved addresses. (Reqwest's `resolve_to_addrs` pins only the
69/// hosts known at build time, so auto-follow would leave redirect hops
70/// unpinned.) A consequence is that connection pooling cannot be shared across
71/// hosts via one long-lived client without weakening per-URL IP pinning, so we
72/// deliberately do not cache clients — SSRF safety wins over pool reuse.
73///
74/// Note that IP pinning only takes effect on a direct connection: when
75/// `HTTP(S)_PROXY` is set, the proxy resolves the host itself and the pinned
76/// addresses are never used. See `docs/product.md`.
77fn build_client(
78    url: &reqwest::Url,
79    timeout: Duration,
80    pinned: &[SocketAddr],
81    tls: &TlsConfig,
82) -> anyhow::Result<Client> {
83    let mut builder = Client::builder()
84        .timeout(timeout)
85        .redirect(Policy::none())
86        .user_agent(USER_AGENT)
87        .gzip(true)
88        .brotli(true);
89
90    // Trust the OS store (+ SSL_CERT_FILE / --ca-cert) so org/proxy root CAs
91    // are accepted, instead of only the bundled webpki roots.
92    builder = tls.apply(builder)?;
93
94    if let Some(host) = url.host_str() {
95        if !pinned.is_empty() {
96            builder = builder.resolve_to_addrs(host, pinned);
97        }
98    }
99    Ok(builder.build()?)
100}
101
102/// One request attempt. The bool in the error reports whether the failure is
103/// transient (worth retrying): connection/timeout errors, 5xx, and 429.
104async fn attempt(client: &Client, url: &str) -> Result<Hop, (anyhow::Error, bool)> {
105    let resp = match client
106        .get(url)
107        .header("Accept", "text/html,application/xhtml+xml,*/*;q=0.8")
108        .header("Accept-Language", "en-US,en;q=0.9")
109        .send()
110        .await
111    {
112        Ok(r) => r,
113        Err(e) => {
114            let transient = transient_send_error(&e);
115            return Err((e.into(), transient));
116        }
117    };
118
119    let status = resp.status();
120
121    // Redirects are surfaced to the caller (which re-validates and pins the
122    // target) rather than followed by reqwest.
123    if status.is_redirection() {
124        return match resp.headers().get(LOCATION).and_then(|v| v.to_str().ok()) {
125            Some(loc) => Ok(Hop::Redirect(loc.to_string())),
126            None => Err((
127                anyhow::anyhow!("redirect ({status}) without a Location header"),
128                false,
129            )),
130        };
131    }
132
133    let resp = match resp.error_for_status() {
134        Ok(r) => r,
135        Err(e) => {
136            let transient = transient_status(status);
137            return Err((e.into(), transient));
138        }
139    };
140
141    let final_url = resp.url().to_string();
142    let content_type = resp
143        .headers()
144        .get(CONTENT_TYPE)
145        .and_then(|v| v.to_str().ok())
146        .map(|s| s.to_string());
147
148    // Decode with the response's declared charset rather than assuming UTF-8:
149    // a windows-1252 / ISO-8859-1 page is otherwise returned full of
150    // replacement characters.
151    let raw = read_body_capped_bytes(resp).await?;
152    let declared = content_type
153        .as_deref()
154        .and_then(charset::from_content_type)
155        .or_else(|| sniff_meta_charset(&raw));
156    let (body, undecodable_charset) = charset::decode(&raw, declared.as_deref());
157
158    Ok(Hop::Page(FetchedPage {
159        body,
160        final_url,
161        content_type,
162        undecodable_charset,
163    }))
164}
165
166/// Issue one hop's request, retrying transient failures with exponential
167/// backoff (200ms, 400ms) while the overall deadline allows.
168async fn fetch_with_retries(client: &Client, url: &str, deadline: Instant) -> anyhow::Result<Hop> {
169    let mut delay = Duration::from_millis(200);
170    for attempt_no in 1..=MAX_ATTEMPTS {
171        match attempt(client, url).await {
172            Ok(hop) => return Ok(hop),
173            Err((err, transient)) => {
174                if attempt_no == MAX_ATTEMPTS || !transient {
175                    return Err(err);
176                }
177                if Instant::now() + delay >= deadline {
178                    return Err(err);
179                }
180                tokio::time::sleep(delay).await;
181                delay *= 2;
182            }
183        }
184    }
185    unreachable!("loop returns on the final attempt")
186}
187
188/// Fetch a URL, following redirects manually so the SSRF guard re-validates and
189/// re-pins each hop (closing the DNS-rebinding window for redirected hosts too),
190/// retrying transient failures with exponential backoff. Caps the redirect
191/// chain at [`MAX_REDIRECTS`], the body at
192/// [`webfetch_core::http::MAX_BODY_BYTES`], and the whole operation at
193/// [`TOTAL_BUDGET_MULTIPLIER`] times `timeout_secs`.
194pub async fn fetch_page(
195    url: &str,
196    timeout_secs: u64,
197    tls: &TlsConfig,
198) -> anyhow::Result<FetchedPage> {
199    let per_request = Duration::from_secs(timeout_secs);
200    let deadline = Instant::now() + per_request * TOTAL_BUDGET_MULTIPLIER;
201
202    let mut current = reqwest::Url::parse(url)?;
203    let mut hops = 0usize;
204
205    loop {
206        let remaining = deadline.saturating_duration_since(Instant::now());
207        if remaining.is_zero() {
208            anyhow::bail!(
209                "fetch exceeded its total budget ({}s across redirects and retries)",
210                timeout_secs * TOTAL_BUDGET_MULTIPLIER as u64
211            );
212        }
213
214        // Validate + resolve the host for THIS hop, then pin the connection to
215        // exactly those addresses.
216        let pinned = guard::validate_url(&current).await?;
217        let client = build_client(&current, per_request.min(remaining), &pinned, tls)?;
218
219        match fetch_with_retries(&client, current.as_str(), deadline).await? {
220            Hop::Page(page) => return Ok(page),
221            Hop::Redirect(location) => {
222                hops += 1;
223                if hops > MAX_REDIRECTS {
224                    anyhow::bail!("too many redirects (>{MAX_REDIRECTS})");
225                }
226                current = current
227                    .join(&location)
228                    .map_err(|e| anyhow::anyhow!("invalid redirect target `{location}`: {e}"))?;
229            }
230        }
231    }
232}
webfetch/fetch.rs

webfetch/
fetch.rs
