webtools-core 0.7.0

Shared primitives: compression, token budgeting, and reference-style URLs
Documentation
//! The HTTP seam the libraries send requests through.
//!
//! Everything above this module builds an [`HttpRequest`], hands it to a
//! [`Transport`], and reads an [`HttpResponse`] back. The native implementation
//! is [`ReqwestTransport`] (behind the `native` feature). The WASI 0.2
//! implementation is [`WasiTransport`] (behind the `wasi` feature), which sends
//! through `wasi:http/outgoing-handler`. A host with its own HTTP stack
//! implements [`Transport`] itself and calls the same generic entry points.
//!
//! One `send` is one hop: a transport never follows redirects. The 3xx response
//! is returned as-is so the caller can re-validate the target before issuing the
//! next request. Bodies are capped at [`crate::http::MAX_BODY_BYTES`].
//!
//! The transport owns the wait between retries too ([`Transport::sleep`]), so a
//! host without a tokio runtime can drive the same retry loop.

use std::fmt;
use std::future::Future;
use std::time::Duration;

#[cfg(feature = "native")]
use crate::http::{push_capped, ERROR_BODY_CAP, MAX_BODY_BYTES, USER_AGENT};
#[cfg(feature = "native")]
use crate::tls::TlsConfig;
#[cfg(feature = "wasi")]
pub use crate::wasi_transport::WasiTransport;

/// HTTP method. The libraries only ever need these two.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Method {
    Get,
    Post,
}

/// One request, owned so it can be retried or re-sent after a redirect.
#[derive(Debug, Clone)]
pub struct HttpRequest {
    pub method: Method,
    pub url: String,
    pub headers: Vec<(String, String)>,
    pub body: Option<Vec<u8>>,
    /// Upper bound for this one request, covering connect, headers and body.
    /// `None` leaves the transport's own default in place.
    pub timeout: Option<Duration>,
}

/// One response. The body is already read and capped.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HttpResponse {
    pub status: u16,
    /// Header names are lowercase.
    pub headers: Vec<(String, String)>,
    pub body: Vec<u8>,
}

impl HttpResponse {
    /// First value of a header, matched case-insensitively.
    pub fn header(&self, name: &str) -> Option<&str> {
        self.headers
            .iter()
            .find(|(k, _)| k.eq_ignore_ascii_case(name))
            .map(|(_, v)| v.as_str())
    }
}

/// Why a request produced no response.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TransportError {
    /// The request did not finish within its timeout. Carries the URL.
    Timeout(String),
    /// The connection could not be made or the request could not be sent.
    Connect(String),
    /// The TLS handshake or certificate check failed.
    Tls(String),
    /// The body exceeded the cap and the transport refuses to truncate it.
    /// The shipped transports truncate instead and never return this.
    BodyTooLarge,
    /// Anything else, with the transport's own description.
    Other(String),
}

impl TransportError {
    /// Is the failure worth retrying? Timeouts and connection-level failures
    /// are; TLS, oversize and other errors are final. This is the classification
    /// the native path has always used.
    pub fn is_transient(&self) -> bool {
        matches!(
            self,
            TransportError::Timeout(_) | TransportError::Connect(_)
        )
    }
}

impl fmt::Display for TransportError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            TransportError::Timeout(url) => write!(f, "request timed out for url ({url})"),
            TransportError::Connect(m) => write!(f, "{m}"),
            TransportError::Tls(m) => write!(f, "TLS error: {m}"),
            TransportError::BodyTooLarge => write!(f, "response body exceeds the size cap"),
            TransportError::Other(m) => write!(f, "{m}"),
        }
    }
}

impl std::error::Error for TransportError {}

/// Sends one HTTP request and returns its response, without following redirects.
pub trait Transport: Send + Sync {
    fn send(
        &self,
        req: HttpRequest,
    ) -> impl Future<Output = Result<HttpResponse, TransportError>> + Send;

    /// Wait for `d`. Every retry backoff goes through here, so the host's clock
    /// decides how the wait happens (tokio timer natively, a WASI pollable on
    /// `wasip2`, or an instant no-op in tests).
    fn sleep(&self, d: Duration) -> impl Future<Output = ()> + Send;
}

/// Lets a borrowed transport be used wherever an owned one is expected.
impl<T: Transport + ?Sized> Transport for &T {
    fn send(
        &self,
        req: HttpRequest,
    ) -> impl Future<Output = Result<HttpResponse, TransportError>> + Send {
        (**self).send(req)
    }

    fn sleep(&self, d: Duration) -> impl Future<Output = ()> + Send {
        (**self).sleep(d)
    }
}

/// The native transport: reqwest with the shared user agent, TLS trust from
/// [`TlsConfig`], gzip and brotli decoding, and redirects disabled.
///
/// The client is built once per instance. Callers that pin a host to specific
/// addresses (the fetch path's DNS-rebinding guard) build one instance per hop
/// with [`ReqwestTransport::pinned`].
#[cfg(feature = "native")]
#[derive(Debug, Clone)]
pub struct ReqwestTransport {
    client: reqwest::Client,
}

#[cfg(feature = "native")]
impl ReqwestTransport {
    /// A transport with the shared client options and the given TLS trust.
    pub fn new(tls: &TlsConfig) -> Result<Self, TransportError> {
        Self::build(tls, None)
    }

    /// As [`ReqwestTransport::new`], but `host` resolves only to `addrs`. An
    /// empty `addrs` (the guard is disabled) leaves resolution alone.
    pub fn pinned(
        tls: &TlsConfig,
        host: &str,
        addrs: &[std::net::SocketAddr],
    ) -> Result<Self, TransportError> {
        Self::build(tls, Some((host, addrs)).filter(|(_, a)| !a.is_empty()))
    }

    fn build(
        tls: &TlsConfig,
        pin: Option<(&str, &[std::net::SocketAddr])>,
    ) -> Result<Self, TransportError> {
        let mut builder = reqwest::Client::builder()
            .redirect(reqwest::redirect::Policy::none())
            .user_agent(USER_AGENT)
            .gzip(true)
            .brotli(true);
        builder = tls
            .apply(builder)
            .map_err(|e| TransportError::Other(format!("{e:#}")))?;
        if let Some((host, addrs)) = pin {
            builder = builder.resolve_to_addrs(host, addrs);
        }
        let client = builder
            .build()
            .map_err(|e| TransportError::Other(format!("building HTTP client: {e}")))?;
        Ok(Self { client })
    }
}

/// The reqwest error with its whole source chain, e.g. `error sending request
/// for url (...): client error (Connect): tcp connect error: Connection refused`.
/// reqwest's own Display stops at the top level and hides the cause.
#[cfg(feature = "native")]
fn error_chain(e: reqwest::Error) -> String {
    format!("{:#}", anyhow::Error::from(e))
}

#[cfg(feature = "native")]
fn map_send_error(e: reqwest::Error, url: &str) -> TransportError {
    if e.is_timeout() {
        TransportError::Timeout(url.to_string())
    } else if e.is_connect() || e.is_request() {
        TransportError::Connect(error_chain(e))
    } else {
        TransportError::Other(error_chain(e))
    }
}

/// Read a body under `cap`. Oversized bodies are truncated, not rejected,
/// which is the behaviour the fetch and search paths have always had.
#[cfg(feature = "native")]
async fn read_capped(mut resp: reqwest::Response, cap: usize) -> Result<Vec<u8>, TransportError> {
    let mut buf: Vec<u8> = Vec::new();
    // Honour Content-Length to pre-size, but never trust it past the cap.
    if let Some(len) = resp.content_length() {
        buf.reserve(len.min(cap as u64) as usize);
    }
    loop {
        match resp.chunk().await {
            Ok(Some(chunk)) => {
                if push_capped(&mut buf, &chunk, cap) {
                    break;
                }
            }
            Ok(None) => break,
            Err(e) if e.is_timeout() => {
                return Err(TransportError::Timeout(resp.url().to_string()))
            }
            Err(e) => return Err(TransportError::Other(error_chain(e))),
        }
    }
    Ok(buf)
}

#[cfg(feature = "native")]
impl Transport for ReqwestTransport {
    async fn send(&self, req: HttpRequest) -> Result<HttpResponse, TransportError> {
        let mut builder = match req.method {
            Method::Get => self.client.get(&req.url),
            Method::Post => self.client.post(&req.url),
        };
        for (name, value) in &req.headers {
            builder = builder.header(name.as_str(), value.as_str());
        }
        if let Some(body) = req.body {
            builder = builder.body(body);
        }
        if let Some(timeout) = req.timeout {
            builder = builder.timeout(timeout);
        }

        let resp = builder
            .send()
            .await
            .map_err(|e| map_send_error(e, &req.url))?;
        let status = resp.status().as_u16();
        let headers = resp
            .headers()
            .iter()
            .filter_map(|(k, v)| Some((k.as_str().to_string(), v.to_str().ok()?.to_string())))
            .collect();
        // A redirect's body is never used: the caller follows the Location.
        // Error bodies are kept only for the message, so they get a small cap.
        let body = match status {
            300..=399 => Vec::new(),
            400..=599 => read_capped(resp, ERROR_BODY_CAP).await?,
            _ => read_capped(resp, MAX_BODY_BYTES).await?,
        };
        Ok(HttpResponse {
            status,
            headers,
            body,
        })
    }

    async fn sleep(&self, d: Duration) {
        tokio::time::sleep(d).await;
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn header_lookup_is_case_insensitive() {
        let resp = HttpResponse {
            status: 302,
            headers: vec![("location".into(), "/next".into())],
            body: Vec::new(),
        };
        assert_eq!(resp.header("Location"), Some("/next"));
        assert_eq!(resp.header("content-type"), None);
    }

    #[test]
    fn only_timeouts_and_connection_failures_are_transient() {
        assert!(TransportError::Timeout("https://example.test/".into()).is_transient());
        assert!(TransportError::Connect("refused".into()).is_transient());
        assert!(!TransportError::Tls("unknown issuer".into()).is_transient());
        assert!(!TransportError::BodyTooLarge.is_transient());
        assert!(!TransportError::Other("bad header".into()).is_transient());
    }

    #[cfg(feature = "native")]
    #[test]
    fn native_transport_builds_from_default_tls() {
        assert!(ReqwestTransport::new(&TlsConfig::default()).is_ok());
    }

    /// A closed local port: the error must keep the OS cause, not just
    /// reqwest's top-level "error sending request" text.
    #[cfg(feature = "native")]
    #[tokio::test]
    async fn connection_refused_keeps_the_cause() {
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let port = listener.local_addr().unwrap().port();
        drop(listener);
        let transport = ReqwestTransport::new(&TlsConfig::default()).unwrap();
        let err = transport
            .send(HttpRequest {
                method: Method::Get,
                url: format!("http://127.0.0.1:{port}/"),
                headers: Vec::new(),
                body: None,
                timeout: Some(Duration::from_secs(5)),
            })
            .await
            .unwrap_err();
        assert!(matches!(err, TransportError::Connect(_)), "{err:?}");
        assert!(err.to_string().contains("Connection refused"), "{err}");
    }
}