webtools-core 0.7.0

Shared primitives: compression, token budgeting, and reference-style URLs
Documentation
//! HTTP primitives shared by the fetch and search paths: the user agent, the
//! response-body cap, and the retry classification. Both paths previously
//! carried their own copy of the last two and had drifted apart — search read
//! bodies without any cap at all.
//!
//! Nothing here touches a concrete HTTP client; the transport-level pieces live
//! in [`crate::transport`].

/// The user agent both paths send.
///
/// A tool-shaped agent (`webfetch/0.1.x`) is refused or challenged by a large
/// share of CDNs, which turns an ordinary fetch into an empty page. The search
/// path already sent a browser agent for exactly this reason; the fetch path
/// did not, and the mismatch was the single most common cause of a 403.
pub const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36";

/// Hard cap on the response body we will read (5 MiB). The HTML extractor turns
/// a page into a few KB of text, so a multi-megabyte body is almost never worth
/// the bandwidth, memory, and parse time — and an unbounded read is a DoS lever.
/// Bodies over the cap are *truncated* (not errored): partial content is still
/// useful and the extractor copes with truncated HTML.
///
/// The cap counts bytes *after* transparent gzip/brotli decoding, so it also
/// bounds a decompression bomb.
pub const MAX_BODY_BYTES: usize = 5 * 1024 * 1024;

/// Cap on a 4xx or 5xx body (64 KiB). Error bodies only feed a message, and
/// the provider's own explanation fits well inside this.
pub const ERROR_BODY_CAP: usize = 64 * 1024;

/// Append as much of `chunk` to `buf` as fits under `max`. Returns `true` once
/// the cap is reached (the body is truncated and the caller should stop).
#[cfg_attr(not(any(feature = "native", feature = "wasi")), allow(dead_code))]
pub(crate) fn push_capped(buf: &mut Vec<u8>, chunk: &[u8], max: usize) -> bool {
    let remaining = max.saturating_sub(buf.len());
    if chunk.len() >= remaining {
        buf.extend_from_slice(&chunk[..remaining]);
        true
    } else {
        buf.extend_from_slice(chunk);
        false
    }
}

/// Is a response status worth retrying? Server errors and explicit throttling.
pub fn transient_status(status: u16) -> bool {
    (500..600).contains(&status) || status == 429
}

/// A status code and its reason phrase, as `404 Not Found`. A code with no
/// standard phrase is shown as the number alone.
///
/// Native builds take the phrase from reqwest's `StatusCode`, so every
/// registered code is covered. The WASI build has no HTTP crate of its own and
/// uses a table of the codes the libraries meet in practice.
pub fn status_text(status: u16) -> String {
    #[cfg(feature = "native")]
    let reason = reqwest::StatusCode::from_u16(status)
        .ok()
        .and_then(|code| code.canonical_reason());
    #[cfg(not(feature = "native"))]
    let reason = wasi_reason(status);
    match reason {
        Some(reason) => format!("{status} {reason}"),
        None => status.to_string(),
    }
}

#[cfg(not(feature = "native"))]
fn wasi_reason(status: u16) -> Option<&'static str> {
    Some(match status {
        200 => "OK",
        204 => "No Content",
        301 => "Moved Permanently",
        302 => "Found",
        303 => "See Other",
        304 => "Not Modified",
        307 => "Temporary Redirect",
        308 => "Permanent Redirect",
        400 => "Bad Request",
        401 => "Unauthorized",
        403 => "Forbidden",
        404 => "Not Found",
        405 => "Method Not Allowed",
        408 => "Request Timeout",
        409 => "Conflict",
        410 => "Gone",
        413 => "Payload Too Large",
        415 => "Unsupported Media Type",
        422 => "Unprocessable Entity",
        429 => "Too Many Requests",
        500 => "Internal Server Error",
        501 => "Not Implemented",
        502 => "Bad Gateway",
        503 => "Service Unavailable",
        504 => "Gateway Timeout",
        _ => return None,
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn status_text_names_the_reason_phrase() {
        assert_eq!(status_text(404), "404 Not Found");
        assert_eq!(status_text(401), "401 Unauthorized");
        // No standard phrase: the number alone.
        assert_eq!(status_text(799), "799");
    }

    #[test]
    fn push_capped_truncates_oversized_chunk() {
        let mut buf = Vec::new();
        // A single chunk larger than the cap is clipped to the cap.
        let stopped = push_capped(&mut buf, &[b'x'; 10], 4);
        assert!(stopped);
        assert_eq!(buf.len(), 4);
    }

    #[test]
    fn push_capped_accumulates_until_cap() {
        let mut buf = Vec::new();
        assert!(!push_capped(&mut buf, b"abc", 8));
        assert!(!push_capped(&mut buf, b"de", 8));
        assert_eq!(buf, b"abcde");
        // Next chunk crosses the cap: only the remaining 3 bytes are kept.
        let stopped = push_capped(&mut buf, b"fghij", 8);
        assert!(stopped);
        assert_eq!(buf.len(), 8);
        assert_eq!(buf, b"abcdefgh");
    }

    #[test]
    fn push_capped_small_body_unaffected() {
        let mut buf = Vec::new();
        let stopped = push_capped(&mut buf, b"hello", 1024);
        assert!(!stopped);
        assert_eq!(buf, b"hello");
    }

    #[test]
    fn transient_status_covers_5xx_and_429() {
        assert!(transient_status(500));
        assert!(transient_status(503));
        assert!(transient_status(429));
        assert!(!transient_status(404));
        assert!(!transient_status(403));
        assert!(!transient_status(202));
    }
}