use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::sync::Arc;
use std::time::Duration;
use reqwest::header::LOCATION;
use reqwest::redirect::Policy;
use reqwest::Response;
use crate::domain::services::content_extractor::ContentExtractionFacade;
use crate::infrastructure::dns::create_ipv4_only_resolver;
use crate::infrastructure::security::ssrf::{SsrfValidator, ValidatedUrl};
use super::error::AgentLibError;
use super::EgressGuard;
pub const DEFAULT_MAX_BYTES: usize = 5 * 1024 * 1024;
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(20);
const MAX_REDIRECTS: u8 = 10;
#[derive(Clone)]
pub struct FetchOptions {
pub max_bytes: usize,
pub timeout: Duration,
pub egress: Option<Arc<dyn EgressGuard>>,
pub follow_redirects: bool,
}
impl Default for FetchOptions {
fn default() -> Self {
Self {
max_bytes: DEFAULT_MAX_BYTES,
timeout: DEFAULT_TIMEOUT,
egress: None,
follow_redirects: true,
}
}
}
#[derive(Debug, Clone)]
pub struct FetchedContent {
pub markdown: String,
pub title: String,
pub resolved_url: String,
pub peer_addr: Option<String>,
}
pub async fn fetch(url: &str, opts: &FetchOptions) -> Result<FetchedContent, AgentLibError> {
let parsed = parse_request_url(url)?;
let validator = SsrfValidator::new();
let mut current = parsed;
let mut redirects: u8 = 0;
#[allow(unused_assignments)]
let mut final_peer_addr: Option<String> = None;
loop {
let (resp, peer_addr) = send_hop(&validator, opts, ¤t).await?;
final_peer_addr = peer_addr;
if opts.follow_redirects && resp.status().is_redirection() {
redirects += 1;
current = next_redirect_target(¤t, &resp, redirects)?;
continue;
}
return finish_response(resp, ¤t, final_peer_addr, opts.max_bytes).await;
}
}
fn parse_request_url(url: &str) -> Result<url::Url, AgentLibError> {
let parsed =
url::Url::parse(url).map_err(|e| AgentLibError::InvalidUrl(format!("{}: {}", url, e)))?;
if parsed.scheme() != "http" && parsed.scheme() != "https" {
return Err(AgentLibError::InvalidUrl(format!(
"unsupported scheme '{}' for {}",
parsed.scheme(),
url
)));
}
Ok(parsed)
}
async fn send_hop(
validator: &SsrfValidator,
opts: &FetchOptions,
current: &url::Url,
) -> Result<(reqwest::Response, Option<String>), AgentLibError> {
if let Some(guard) = &opts.egress {
if !guard.allow(current) {
return Err(AgentLibError::EgressDenied {
url: current.to_string(),
});
}
}
let client = build_hop_client(validator, opts, current).await?;
let resp = client
.get(current.clone())
.send()
.await
.map_err(|e| map_reqwest_error(current.as_str(), e))?;
let peer_addr = resp.remote_addr().map(|a| a.to_string());
Ok((resp, peer_addr))
}
async fn finish_response(
resp: reqwest::Response,
current: &url::Url,
peer_addr: Option<String>,
max_bytes: usize,
) -> Result<FetchedContent, AgentLibError> {
if !resp.status().is_success() {
return Err(AgentLibError::HttpStatus {
status: resp.status().as_u16(),
url: current.to_string(),
});
}
let html = read_body_limited(resp, max_bytes).await?;
convert_to_markdown(&html, current, peer_addr).await
}
async fn build_hop_client(
validator: &SsrfValidator,
opts: &FetchOptions,
current: &url::Url,
) -> Result<reqwest::Client, AgentLibError> {
if opts.egress.is_some() {
build_plain_client(opts.timeout)
} else {
let validated = validator.validate(current.as_str()).await.map_err(|e| {
AgentLibError::SsrfDenied {
url: current.to_string(),
reason: e.kind().to_string(),
}
})?;
build_pinned_client(&validated, opts.timeout)
}
}
fn next_redirect_target(
current: &url::Url,
resp: &Response,
redirects: u8,
) -> Result<url::Url, AgentLibError> {
if redirects > MAX_REDIRECTS {
return Err(AgentLibError::TooManyRedirects {
max_redirects: MAX_REDIRECTS,
});
}
let location = resp
.headers()
.get(LOCATION)
.and_then(|v| v.to_str().ok())
.ok_or_else(|| {
AgentLibError::Network(format!(
"redirect {} missing Location header",
resp.status()
))
})?;
current
.join(location)
.map_err(|e| AgentLibError::InvalidUrl(format!("bad redirect: {}", e)))
}
fn build_plain_client(timeout: Duration) -> Result<reqwest::Client, AgentLibError> {
reqwest::Client::builder()
.user_agent(crate::utils::http_client::DEFAULT_USER_AGENT)
.timeout(timeout)
.connect_timeout(crate::utils::http_client::connect_timeout_for(timeout))
.cookie_store(true)
.local_address(Some(Ipv4Addr::UNSPECIFIED.into()))
.dns_resolver(create_ipv4_only_resolver())
.redirect(Policy::none())
.build()
.map_err(|e| AgentLibError::Network(format!("failed to build client: {}", e)))
}
fn build_pinned_client(
validated: &ValidatedUrl,
timeout: Duration,
) -> Result<reqwest::Client, AgentLibError> {
let host = validated.parsed_url.host_str().unwrap_or("").to_string();
let port = validated.port;
let resolve_addrs: Vec<SocketAddr> = validated
.resolved_ips
.iter()
.map(|ip| SocketAddr::new(*ip, port))
.collect();
if resolve_addrs.is_empty() {
return Err(AgentLibError::Internal("SSRF: no resolved IPs".to_string()));
}
let mut builder = reqwest::Client::builder()
.user_agent(crate::utils::http_client::DEFAULT_USER_AGENT)
.timeout(timeout)
.connect_timeout(crate::utils::http_client::connect_timeout_for(timeout))
.cookie_store(true)
.local_address(Some(Ipv4Addr::UNSPECIFIED.into()))
.dns_resolver(create_ipv4_only_resolver())
.redirect(Policy::none());
if !host.is_empty() && host.parse::<IpAddr>().is_err() {
builder = builder.resolve_to_addrs(&host, &resolve_addrs);
}
builder
.build()
.map_err(|e| AgentLibError::Network(format!("failed to build client: {}", e)))
}
async fn read_body_limited(resp: Response, max_bytes: usize) -> Result<String, AgentLibError> {
crate::utils::http_client::read_body_limited(resp, max_bytes)
.await
.map_err(|e| match e {
crate::utils::http_client::BodyReadError::LimitExceeded { max_bytes } => {
AgentLibError::MaxBytesExceeded { max_bytes }
}
crate::utils::http_client::BodyReadError::Network(e) => {
AgentLibError::Network(e.to_string())
}
})
}
async fn convert_to_markdown(
html: &str,
current: &url::Url,
peer_addr: Option<String>,
) -> Result<FetchedContent, AgentLibError> {
let facade = ContentExtractionFacade::new(None);
let markdown_converter = crate::domain::services::markdown::build_converter(
&crate::config::settings::MarkdownSettings::default(),
);
let extracted = facade.extract(html, current.as_str()).await;
let (title, body) = match extracted {
Ok(c) if !c.text.trim().is_empty() => {
let title = c.title.clone().unwrap_or_default();
let md = markdown_converter
.convert(&c.text)
.map_err(|e| AgentLibError::Markdown(e.to_string()))?;
(title, md)
}
_ => {
let md = markdown_converter
.convert(html)
.map_err(|e| AgentLibError::Markdown(e.to_string()))?;
(String::new(), md)
}
};
let markdown = if title.is_empty() {
body
} else {
format!("# {}\n\n{}", title, body)
};
Ok(FetchedContent {
markdown,
title,
resolved_url: current.to_string(),
peer_addr,
})
}
fn map_reqwest_error(url: &str, e: reqwest::Error) -> AgentLibError {
if e.is_timeout() {
AgentLibError::Timeout(format!("{}: {}", url, e))
} else if e.is_connect() {
AgentLibError::Network(format!("connect failed for {}: {}", url, e))
} else {
AgentLibError::Network(format!("{}: {}", url, e))
}
}
#[cfg(test)]
mod tests {
use super::*;
use url::Url;
struct PredicateGuard<F: Fn(&Url) -> bool + Send + Sync> {
f: F,
}
impl<F: Fn(&Url) -> bool + Send + Sync> EgressGuard for PredicateGuard<F> {
fn allow(&self, url: &Url) -> bool {
(self.f)(url)
}
}
fn allow_all() -> Arc<dyn EgressGuard> {
Arc::new(PredicateGuard { f: |_| true })
}
fn deny_all() -> Arc<dyn EgressGuard> {
Arc::new(PredicateGuard { f: |_| false })
}
fn deny_path(path: &'static str) -> Arc<dyn EgressGuard> {
Arc::new(PredicateGuard {
f: move |u: &Url| u.path() != path,
})
}
const TEST_HTML: &str = r#"<!DOCTYPE html>
<html>
<head>
<title>Test Page Title</title>
</head>
<body>
<article>
<h1>Main Heading</h1>
<p>This is the first paragraph of meaningful content for extraction.</p>
<p>This is the second paragraph with more detail.</p>
</article>
</body>
</html>"#;
#[tokio::test]
async fn fetch_invalid_url_returns_error_not_panic() {
let err = fetch("not a url", &FetchOptions::default())
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::InvalidUrl(_)));
}
#[tokio::test]
async fn fetch_unsupported_scheme_returns_invalid_url() {
let err = fetch("file:///etc/passwd", &FetchOptions::default())
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::InvalidUrl(_)));
}
#[tokio::test]
async fn fetch_known_html_produces_markdown() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/article"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(TEST_HTML, "text/html"))
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
..Default::default()
};
let content = fetch(&format!("{}/article", server.uri()), &opts)
.await
.expect("fetch should succeed");
assert!(
!content.markdown.trim().is_empty(),
"markdown should be non-empty"
);
assert!(
content.markdown.contains("Main Heading"),
"markdown should contain the title text: {}",
content.markdown
);
}
#[tokio::test]
async fn fetch_peer_addr_backfilled() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/page"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(TEST_HTML, "text/html"))
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
..Default::default()
};
let content = fetch(&format!("{}/page", server.uri()), &opts)
.await
.expect("fetch should succeed");
assert!(
content.peer_addr.is_some(),
"peer_addr should be backfilled from connection metadata"
);
assert!(content.peer_addr.as_deref().unwrap().contains(':'));
}
#[tokio::test]
async fn fetch_egress_deny_returns_egress_denied_before_connection() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/never"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(TEST_HTML, "text/html"))
.expect(0)
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(deny_all()),
..Default::default()
};
let err = fetch(&format!("{}/never", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::EgressDenied { .. }));
}
#[tokio::test]
async fn fetch_302_redirect_to_denied_target_is_rejected_on_second_hop() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/start"))
.respond_with(
wiremock::ResponseTemplate::new(302).insert_header("Location", "/blocked"),
)
.mount(&server)
.await;
wiremock::Mock::given(wiremock::matchers::path("/blocked"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(TEST_HTML, "text/html"))
.expect(0) .mount(&server)
.await;
let opts = FetchOptions {
egress: Some(deny_path("/blocked")),
..Default::default()
};
let err = fetch(&format!("{}/start", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::EgressDenied { .. }));
}
#[tokio::test]
async fn fetch_302_redirect_followed_to_final_content() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/redirect"))
.respond_with(wiremock::ResponseTemplate::new(302).insert_header("Location", "/final"))
.mount(&server)
.await;
wiremock::Mock::given(wiremock::matchers::path("/final"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_raw(TEST_HTML, "text/html"))
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
..Default::default()
};
let content = fetch(&format!("{}/redirect", server.uri()), &opts)
.await
.expect("redirect should be followed");
assert!(content.resolved_url.ends_with("/final"));
assert!(content.markdown.contains("Main Heading"));
}
#[tokio::test]
async fn fetch_follow_redirects_false_returns_http_status() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/redirect"))
.respond_with(wiremock::ResponseTemplate::new(302).insert_header("Location", "/final"))
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
follow_redirects: false,
..Default::default()
};
let err = fetch(&format!("{}/redirect", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::HttpStatus { status: 302, .. }));
}
#[tokio::test]
async fn fetch_max_bytes_exceeded() {
let server = wiremock::MockServer::start().await;
let big_body = "x".repeat(8192);
wiremock::Mock::given(wiremock::matchers::path("/big"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_raw(big_body.as_str(), "text/plain"),
)
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
max_bytes: 1024,
..Default::default()
};
let err = fetch(&format!("{}/big", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(
err,
AgentLibError::MaxBytesExceeded { max_bytes: 1024 }
));
}
#[tokio::test]
async fn fetch_http_error_status() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/notfound"))
.respond_with(wiremock::ResponseTemplate::new(404))
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
..Default::default()
};
let err = fetch(&format!("{}/notfound", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::HttpStatus { status: 404, .. }));
}
#[tokio::test]
async fn fetch_egress_none_runs_crawlrs_ssrf_validation() {
for url in ["http://10.0.0.1/secret", "http://127.0.0.1:9/page"] {
let err = fetch(url, &FetchOptions::default()).await.unwrap_err();
assert!(
matches!(err, AgentLibError::SsrfDenied { .. }),
"private/loopback target {} must yield SsrfDenied",
url
);
}
}
#[tokio::test]
async fn fetch_too_many_redirects() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/loop"))
.respond_with(wiremock::ResponseTemplate::new(302).insert_header("Location", "/loop"))
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
..Default::default()
};
let err = fetch(&format!("{}/loop", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::TooManyRedirects { .. }));
}
#[tokio::test]
async fn fetch_timeout_returns_timeout_error() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::path("/slow"))
.respond_with(
wiremock::ResponseTemplate::new(200)
.set_delay(std::time::Duration::from_secs(5))
.set_body_raw(TEST_HTML, "text/html"),
)
.mount(&server)
.await;
let opts = FetchOptions {
egress: Some(allow_all()),
timeout: Duration::from_millis(50),
..Default::default()
};
let err = fetch(&format!("{}/slow", server.uri()), &opts)
.await
.unwrap_err();
assert!(matches!(err, AgentLibError::Timeout(_)), "got {:?}", err);
}
}