Skip to main content

webfetch_core/
transport.rs

1//! The HTTP seam the libraries send requests through.
2//!
3//! Everything above this module builds an [`HttpRequest`], hands it to a
4//! [`Transport`], and reads an [`HttpResponse`] back. The native implementation
5//! is [`ReqwestTransport`] (behind the `native` feature). The WASI 0.2
6//! implementation is [`WasiTransport`] (behind the `wasi` feature), which sends
7//! through `wasi:http/outgoing-handler`. A host with its own HTTP stack
8//! implements [`Transport`] itself and calls the same generic entry points.
9//!
10//! One `send` is one hop: a transport never follows redirects. The 3xx response
11//! is returned as-is so the caller can re-validate the target before issuing the
12//! next request. Bodies are capped at [`crate::http::MAX_BODY_BYTES`].
13//!
14//! The transport owns the wait between retries too ([`Transport::sleep`]), so a
15//! host without a tokio runtime can drive the same retry loop.
16
17use std::fmt;
18use std::future::Future;
19use std::time::Duration;
20
21#[cfg(feature = "native")]
22use crate::http::{push_capped, ERROR_BODY_CAP, MAX_BODY_BYTES, USER_AGENT};
23#[cfg(feature = "native")]
24use crate::tls::TlsConfig;
25#[cfg(feature = "wasi")]
26pub use crate::wasi_transport::WasiTransport;
27
28/// HTTP method. The libraries only ever need these two.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30pub enum Method {
31    Get,
32    Post,
33}
34
35/// One request, owned so it can be retried or re-sent after a redirect.
36#[derive(Debug, Clone)]
37pub struct HttpRequest {
38    pub method: Method,
39    pub url: String,
40    pub headers: Vec<(String, String)>,
41    pub body: Option<Vec<u8>>,
42    /// Upper bound for this one request, covering connect, headers and body.
43    /// `None` leaves the transport's own default in place.
44    pub timeout: Option<Duration>,
45}
46
47/// One response. The body is already read and capped.
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub struct HttpResponse {
50    pub status: u16,
51    /// Header names are lowercase.
52    pub headers: Vec<(String, String)>,
53    pub body: Vec<u8>,
54}
55
56impl HttpResponse {
57    /// First value of a header, matched case-insensitively.
58    pub fn header(&self, name: &str) -> Option<&str> {
59        self.headers
60            .iter()
61            .find(|(k, _)| k.eq_ignore_ascii_case(name))
62            .map(|(_, v)| v.as_str())
63    }
64}
65
66/// Why a request produced no response.
67#[derive(Debug, Clone, PartialEq, Eq)]
68pub enum TransportError {
69    /// The request did not finish within its timeout. Carries the URL.
70    Timeout(String),
71    /// The connection could not be made or the request could not be sent.
72    Connect(String),
73    /// The TLS handshake or certificate check failed.
74    Tls(String),
75    /// The body exceeded the cap and the transport refuses to truncate it.
76    /// The shipped transports truncate instead and never return this.
77    BodyTooLarge,
78    /// Anything else, with the transport's own description.
79    Other(String),
80}
81
82impl TransportError {
83    /// Is the failure worth retrying? Timeouts and connection-level failures
84    /// are; TLS, oversize and other errors are final. This is the classification
85    /// the native path has always used.
86    pub fn is_transient(&self) -> bool {
87        matches!(
88            self,
89            TransportError::Timeout(_) | TransportError::Connect(_)
90        )
91    }
92}
93
94impl fmt::Display for TransportError {
95    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
96        match self {
97            TransportError::Timeout(url) => write!(f, "request timed out for url ({url})"),
98            TransportError::Connect(m) => write!(f, "{m}"),
99            TransportError::Tls(m) => write!(f, "TLS error: {m}"),
100            TransportError::BodyTooLarge => write!(f, "response body exceeds the size cap"),
101            TransportError::Other(m) => write!(f, "{m}"),
102        }
103    }
104}
105
106impl std::error::Error for TransportError {}
107
108/// Sends one HTTP request and returns its response, without following redirects.
109pub trait Transport: Send + Sync {
110    fn send(
111        &self,
112        req: HttpRequest,
113    ) -> impl Future<Output = Result<HttpResponse, TransportError>> + Send;
114
115    /// Wait for `d`. Every retry backoff goes through here, so the host's clock
116    /// decides how the wait happens (tokio timer natively, a WASI pollable on
117    /// `wasip2`, or an instant no-op in tests).
118    fn sleep(&self, d: Duration) -> impl Future<Output = ()> + Send;
119}
120
121/// Lets a borrowed transport be used wherever an owned one is expected.
122impl<T: Transport + ?Sized> Transport for &T {
123    fn send(
124        &self,
125        req: HttpRequest,
126    ) -> impl Future<Output = Result<HttpResponse, TransportError>> + Send {
127        (**self).send(req)
128    }
129
130    fn sleep(&self, d: Duration) -> impl Future<Output = ()> + Send {
131        (**self).sleep(d)
132    }
133}
134
135/// The native transport: reqwest with the shared user agent, TLS trust from
136/// [`TlsConfig`], gzip and brotli decoding, and redirects disabled.
137///
138/// The client is built once per instance. Callers that pin a host to specific
139/// addresses (the fetch path's DNS-rebinding guard) build one instance per hop
140/// with [`ReqwestTransport::pinned`].
141#[cfg(feature = "native")]
142#[derive(Debug, Clone)]
143pub struct ReqwestTransport {
144    client: reqwest::Client,
145}
146
147#[cfg(feature = "native")]
148impl ReqwestTransport {
149    /// A transport with the shared client options and the given TLS trust.
150    pub fn new(tls: &TlsConfig) -> Result<Self, TransportError> {
151        Self::build(tls, None)
152    }
153
154    /// As [`ReqwestTransport::new`], but `host` resolves only to `addrs`. An
155    /// empty `addrs` (the guard is disabled) leaves resolution alone.
156    pub fn pinned(
157        tls: &TlsConfig,
158        host: &str,
159        addrs: &[std::net::SocketAddr],
160    ) -> Result<Self, TransportError> {
161        Self::build(tls, Some((host, addrs)).filter(|(_, a)| !a.is_empty()))
162    }
163
164    fn build(
165        tls: &TlsConfig,
166        pin: Option<(&str, &[std::net::SocketAddr])>,
167    ) -> Result<Self, TransportError> {
168        let mut builder = reqwest::Client::builder()
169            .redirect(reqwest::redirect::Policy::none())
170            .user_agent(USER_AGENT)
171            .gzip(true)
172            .brotli(true);
173        builder = tls
174            .apply(builder)
175            .map_err(|e| TransportError::Other(format!("{e:#}")))?;
176        if let Some((host, addrs)) = pin {
177            builder = builder.resolve_to_addrs(host, addrs);
178        }
179        let client = builder
180            .build()
181            .map_err(|e| TransportError::Other(format!("building HTTP client: {e}")))?;
182        Ok(Self { client })
183    }
184}
185
186/// The reqwest error with its whole source chain, e.g. `error sending request
187/// for url (...): client error (Connect): tcp connect error: Connection refused`.
188/// reqwest's own Display stops at the top level and hides the cause.
189#[cfg(feature = "native")]
190fn error_chain(e: reqwest::Error) -> String {
191    format!("{:#}", anyhow::Error::from(e))
192}
193
194#[cfg(feature = "native")]
195fn map_send_error(e: reqwest::Error, url: &str) -> TransportError {
196    if e.is_timeout() {
197        TransportError::Timeout(url.to_string())
198    } else if e.is_connect() || e.is_request() {
199        TransportError::Connect(error_chain(e))
200    } else {
201        TransportError::Other(error_chain(e))
202    }
203}
204
205/// Read a body under `cap`. Oversized bodies are truncated, not rejected,
206/// which is the behaviour the fetch and search paths have always had.
207#[cfg(feature = "native")]
208async fn read_capped(mut resp: reqwest::Response, cap: usize) -> Result<Vec<u8>, TransportError> {
209    let mut buf: Vec<u8> = Vec::new();
210    // Honour Content-Length to pre-size, but never trust it past the cap.
211    if let Some(len) = resp.content_length() {
212        buf.reserve(len.min(cap as u64) as usize);
213    }
214    loop {
215        match resp.chunk().await {
216            Ok(Some(chunk)) => {
217                if push_capped(&mut buf, &chunk, cap) {
218                    break;
219                }
220            }
221            Ok(None) => break,
222            Err(e) if e.is_timeout() => {
223                return Err(TransportError::Timeout(resp.url().to_string()))
224            }
225            Err(e) => return Err(TransportError::Other(error_chain(e))),
226        }
227    }
228    Ok(buf)
229}
230
231#[cfg(feature = "native")]
232impl Transport for ReqwestTransport {
233    async fn send(&self, req: HttpRequest) -> Result<HttpResponse, TransportError> {
234        let mut builder = match req.method {
235            Method::Get => self.client.get(&req.url),
236            Method::Post => self.client.post(&req.url),
237        };
238        for (name, value) in &req.headers {
239            builder = builder.header(name.as_str(), value.as_str());
240        }
241        if let Some(body) = req.body {
242            builder = builder.body(body);
243        }
244        if let Some(timeout) = req.timeout {
245            builder = builder.timeout(timeout);
246        }
247
248        let resp = builder
249            .send()
250            .await
251            .map_err(|e| map_send_error(e, &req.url))?;
252        let status = resp.status().as_u16();
253        let headers = resp
254            .headers()
255            .iter()
256            .filter_map(|(k, v)| Some((k.as_str().to_string(), v.to_str().ok()?.to_string())))
257            .collect();
258        // A redirect's body is never used: the caller follows the Location.
259        // Error bodies are kept only for the message, so they get a small cap.
260        let body = match status {
261            300..=399 => Vec::new(),
262            400..=599 => read_capped(resp, ERROR_BODY_CAP).await?,
263            _ => read_capped(resp, MAX_BODY_BYTES).await?,
264        };
265        Ok(HttpResponse {
266            status,
267            headers,
268            body,
269        })
270    }
271
272    async fn sleep(&self, d: Duration) {
273        tokio::time::sleep(d).await;
274    }
275}
276
277#[cfg(test)]
278mod tests {
279    use super::*;
280
281    #[test]
282    fn header_lookup_is_case_insensitive() {
283        let resp = HttpResponse {
284            status: 302,
285            headers: vec![("location".into(), "/next".into())],
286            body: Vec::new(),
287        };
288        assert_eq!(resp.header("Location"), Some("/next"));
289        assert_eq!(resp.header("content-type"), None);
290    }
291
292    #[test]
293    fn only_timeouts_and_connection_failures_are_transient() {
294        assert!(TransportError::Timeout("https://example.test/".into()).is_transient());
295        assert!(TransportError::Connect("refused".into()).is_transient());
296        assert!(!TransportError::Tls("unknown issuer".into()).is_transient());
297        assert!(!TransportError::BodyTooLarge.is_transient());
298        assert!(!TransportError::Other("bad header".into()).is_transient());
299    }
300
301    #[cfg(feature = "native")]
302    #[test]
303    fn native_transport_builds_from_default_tls() {
304        assert!(ReqwestTransport::new(&TlsConfig::default()).is_ok());
305    }
306
307    /// A closed local port: the error must keep the OS cause, not just
308    /// reqwest's top-level "error sending request" text.
309    #[cfg(feature = "native")]
310    #[tokio::test]
311    async fn connection_refused_keeps_the_cause() {
312        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
313        let port = listener.local_addr().unwrap().port();
314        drop(listener);
315        let transport = ReqwestTransport::new(&TlsConfig::default()).unwrap();
316        let err = transport
317            .send(HttpRequest {
318                method: Method::Get,
319                url: format!("http://127.0.0.1:{port}/"),
320                headers: Vec::new(),
321                body: None,
322                timeout: Some(Duration::from_secs(5)),
323            })
324            .await
325            .unwrap_err();
326        assert!(matches!(err, TransportError::Connect(_)), "{err:?}");
327        assert!(err.to_string().contains("Connection refused"), "{err}");
328    }
329}