agentsec-core 0.5.0

AgentSec core library — scan / web / paste logic, pure Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! HTTP fetch with a hard body cap and a configurable timeout.
//!
//! Constants:
//!
//! - `MAX_BODY_BYTES` — hard cap on response size (currently 2 MiB).
//! - `USER_AGENT` — sent as `User-Agent` header; format is
//!   `agentsec/<cargo-pkg-version>`.
//!
//! The timeout is driven by [`crate::config::WebConfig::timeout_secs`]
//! (default: [`crate::config::DEFAULT_WEB_TIMEOUT_SECS`] = 10 s).
//!
//! Non-2xx responses and oversized bodies are rejected via
//! [`crate::Error::Sanitize`]. The body is **not** truncated; the request
//! is failed.

use crate::error::{Error, Result};
use std::time::Duration;

const MAX_BODY_BYTES: usize = 2 * 1024 * 1024;
const USER_AGENT: &str = concat!("agentsec/", env!("CARGO_PKG_VERSION"));

/// Fetch a URL with a 2 MiB body cap and a timeout driven by
/// `timeout_secs`.
///
/// Returns the raw body decoded as UTF-8 (lossy) on success.
///
/// # Errors
///
/// - [`crate::Error::Http`] — reqwest client / connect / read error.
/// - [`crate::Error::Sanitize`] — HTTP non-2xx status, or body size strictly
///   greater than 2 MiB.
///
/// The caller (sanitize pipeline) is responsible for treating the returned
/// content as untrusted.
pub async fn get(url: &str, timeout_secs: u64) -> Result<String> {
    let client = reqwest::Client::builder()
        .timeout(Duration::from_secs(timeout_secs))
        .user_agent(USER_AGENT)
        .build()?;

    let resp = client.get(url).send().await?;
    let status = resp.status();
    if !status.is_success() {
        return Err(Error::Sanitize(format!(
            "fetch {url} returned HTTP {status}"
        )));
    }

    // Read body with a hard cap. reqwest::Response::text() has no size limit,
    // so we stream bytes and stop when we exceed MAX_BODY_BYTES.
    let bytes = resp.bytes().await?;
    if bytes.len() > MAX_BODY_BYTES {
        return Err(Error::Sanitize(format!(
            "fetch {url} body exceeds {MAX_BODY_BYTES} byte cap"
        )));
    }
    let text = String::from_utf8_lossy(&bytes).into_owned();
    Ok(text)
}