agentsec_core/web/
fetch.rs

1//! HTTP fetch with a hard body cap and a fixed timeout.
2//!
3//! Constants:
4//!
5//! - `DEFAULT_TIMEOUT_SECS` — request timeout (currently 15 s).
6//! - `MAX_BODY_BYTES` — hard cap on response size (currently 2 MiB).
7//! - `USER_AGENT` — sent as `User-Agent` header; format is
8//!   `agentsec/<cargo-pkg-version>`.
9//!
10//! Non-2xx responses and oversized bodies are rejected via
11//! [`crate::Error::Sanitize`]. The body is **not** truncated; the request
12//! is failed.
13
14use crate::error::{Error, Result};
15use std::time::Duration;
16
17const DEFAULT_TIMEOUT_SECS: u64 = 15;
18const MAX_BODY_BYTES: usize = 2 * 1024 * 1024;
19const USER_AGENT: &str = concat!("agentsec/", env!("CARGO_PKG_VERSION"));
20
21/// Fetch a URL with a 2 MiB body cap and a 15 s timeout.
22///
23/// Returns the raw body decoded as UTF-8 (lossy) on success.
24///
25/// # Errors
26///
27/// - [`crate::Error::Http`] — reqwest client / connect / read error.
28/// - [`crate::Error::Sanitize`] — HTTP non-2xx status, or body size strictly
29///   greater than 2 MiB.
30///
31/// The caller (sanitize pipeline) is responsible for treating the returned
32/// content as untrusted.
33pub async fn get(url: &str) -> Result<String> {
34    let client = reqwest::Client::builder()
35        .timeout(Duration::from_secs(DEFAULT_TIMEOUT_SECS))
36        .user_agent(USER_AGENT)
37        .build()?;
38
39    let resp = client.get(url).send().await?;
40    let status = resp.status();
41    if !status.is_success() {
42        return Err(Error::Sanitize(format!(
43            "fetch {url} returned HTTP {status}"
44        )));
45    }
46
47    // Read body with a hard cap. reqwest::Response::text() has no size limit,
48    // so we stream bytes and stop when we exceed MAX_BODY_BYTES.
49    let bytes = resp.bytes().await?;
50    if bytes.len() > MAX_BODY_BYTES {
51        return Err(Error::Sanitize(format!(
52            "fetch {url} body exceeds {MAX_BODY_BYTES} byte cap"
53        )));
54    }
55    let text = String::from_utf8_lossy(&bytes).into_owned();
56    Ok(text)
57}
agentsec_core/web/fetch.rs

agentsec_core/web/
fetch.rs
