agentsec_core/web/
fetch.rs

1//! HTTP fetch with a hard body cap and a configurable timeout.
2//!
3//! Constants:
4//!
5//! - `MAX_BODY_BYTES` — hard cap on response size (currently 2 MiB).
6//! - `USER_AGENT` — sent as `User-Agent` header; format is
7//!   `agentsec/<cargo-pkg-version>`.
8//!
9//! The timeout is driven by [`crate::config::WebConfig::timeout_secs`]
10//! (default: [`crate::config::DEFAULT_WEB_TIMEOUT_SECS`] = 10 s).
11//!
12//! Non-2xx responses and oversized bodies are rejected via
13//! [`crate::Error::Sanitize`]. The body is **not** truncated; the request
14//! is failed.
15
16use crate::error::{Error, Result};
17use std::time::Duration;
18
19const MAX_BODY_BYTES: usize = 2 * 1024 * 1024;
20const USER_AGENT: &str = concat!("agentsec/", env!("CARGO_PKG_VERSION"));
21
22/// Fetch a URL with a 2 MiB body cap and a timeout driven by
23/// `timeout_secs`.
24///
25/// Returns the raw body decoded as UTF-8 (lossy) on success.
26///
27/// # Errors
28///
29/// - [`crate::Error::Http`] — reqwest client / connect / read error.
30/// - [`crate::Error::Sanitize`] — HTTP non-2xx status, or body size strictly
31///   greater than 2 MiB.
32///
33/// The caller (sanitize pipeline) is responsible for treating the returned
34/// content as untrusted.
35pub async fn get(url: &str, timeout_secs: u64) -> Result<String> {
36    let client = reqwest::Client::builder()
37        .timeout(Duration::from_secs(timeout_secs))
38        .user_agent(USER_AGENT)
39        .build()?;
40
41    let resp = client.get(url).send().await?;
42    let status = resp.status();
43    if !status.is_success() {
44        return Err(Error::Sanitize(format!(
45            "fetch {url} returned HTTP {status}"
46        )));
47    }
48
49    // Read body with a hard cap. reqwest::Response::text() has no size limit,
50    // so we stream bytes and stop when we exceed MAX_BODY_BYTES.
51    let bytes = resp.bytes().await?;
52    if bytes.len() > MAX_BODY_BYTES {
53        return Err(Error::Sanitize(format!(
54            "fetch {url} body exceeds {MAX_BODY_BYTES} byte cap"
55        )));
56    }
57    let text = String::from_utf8_lossy(&bytes).into_owned();
58    Ok(text)
59}
agentsec_core/web/fetch.rs

agentsec_core/web/
fetch.rs
