use crate::client::FetchOptions;
use crate::content::{
ContentFocus, ContentOutputFormat, ContentProcessorInput, ContentProcessorRegistry,
HtmlProcessor,
};
use crate::convert::{
classify_agent_resource, filter_excessive_newlines, is_html, is_markdown_content_type,
is_plain_text_content_type,
};
use crate::documents::{DefaultFormatDetector, FormatDetector};
use crate::error::FetchError;
use crate::fetchers::Fetcher;
use crate::file_saver::FileSaver;
use crate::transport::{BodyStream, TransportMethod, TransportRequest, TransportResponse};
use crate::types::{
AgentResource, FetchRequest, FetchResponse, HttpMethod, PageMetadata, PageQuality,
};
use crate::DEFAULT_USER_AGENT;
use async_trait::async_trait;
use bytes::Bytes;
use futures::StreamExt;
use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_DISPOSITION, LOCATION, USER_AGENT};
use std::collections::HashSet;
use std::time::Duration;
use tracing::{debug, error, warn};
use url::Url;
pub(crate) fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> {
headers
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case(name))
.map(|(_, v)| v.as_str())
}
fn headers_to_pairs(headers: &HeaderMap) -> Vec<(String, String)> {
headers
.iter()
.filter_map(|(name, value)| {
value
.to_str()
.ok()
.map(|v| (name.as_str().to_string(), v.to_string()))
})
.collect()
}
pub(crate) fn pinned_addrs_for_url(
url: &Url,
options: &FetchOptions,
) -> Result<Vec<std::net::SocketAddr>, FetchError> {
let Some(host) = url.host_str() else {
return Ok(Vec::new());
};
let port = url.port_or_known_default().unwrap_or(80);
options
.dns_policy
.pinned_addrs(host, port)
.map_err(|_| FetchError::BlockedUrl)
}
const BINARY_PREFIXES: &[&str] = &[
"image/",
"audio/",
"video/",
"application/octet-stream",
"application/pdf",
"application/x-pdf",
"application/zip",
"application/gzip",
"application/x-tar",
"application/x-rar",
"application/x-7z",
"application/vnd.ms-",
"application/vnd.openxmlformats",
"application/vnd.oasis.opendocument",
"application/msword",
"application/epub+zip",
"font/",
];
const FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(1);
pub(crate) const BODY_TIMEOUT: Duration = Duration::from_secs(30);
#[cfg(feature = "render-rakers")]
const RAKERS_SCRIPT_TIMEOUT: Duration = Duration::from_secs(5);
pub(crate) const TRUNCATION_MESSAGE: &str = "\n\n[..content truncated...]";
const MAX_REDIRECTS: usize = 10;
const AGENT_RESOURCE_PROBES: &[(&str, &str)] = &[
("/llms.txt", "llms-txt"),
("/llms-full.txt", "llms-full-txt"),
("/auth.md", "auth"),
("/.well-known/oauth-authorization-server", "oauth"),
("/.well-known/openid-configuration", "openid"),
(
"/.well-known/oauth-protected-resource",
"oauth-protected-resource",
),
("/.well-known/api-catalog", "api-catalog"),
("/.well-known/mcp/server-card.json", "mcp"),
("/.well-known/agent-card.json", "agent-card"),
("/.well-known/agent-skills/index.json", "agent-skills"),
];
const AGENT_PROBE_TIMEOUT: Duration = Duration::from_secs(3);
pub(crate) const DEFAULT_MAX_BODY_SIZE: usize = 10 * 1024 * 1024;
pub struct DefaultFetcher {
content_processors: ContentProcessorRegistry,
}
impl DefaultFetcher {
pub fn new() -> Self {
Self::with_content_processors(ContentProcessorRegistry::with_defaults())
}
pub fn with_content_processors(mut content_processors: ContentProcessorRegistry) -> Self {
if !content_processors.contains_named("html") {
content_processors.register(Box::new(HtmlProcessor::default()));
}
Self { content_processors }
}
}
impl Default for DefaultFetcher {
fn default() -> Self {
Self::new()
}
}
pub(crate) fn build_headers(
options: &FetchOptions,
accept: &str,
request: &FetchRequest,
) -> HeaderMap {
let mut headers = HeaderMap::new();
let user_agent = options.user_agent.as_deref().unwrap_or(DEFAULT_USER_AGENT);
headers.insert(
USER_AGENT,
HeaderValue::from_str(user_agent)
.unwrap_or_else(|_| HeaderValue::from_static(DEFAULT_USER_AGENT)),
);
headers.insert(
ACCEPT,
HeaderValue::from_str(accept).unwrap_or_else(|_| HeaderValue::from_static("*/*")),
);
if let Some(ref etag) = request.if_none_match {
if let Ok(v) = HeaderValue::from_str(etag) {
headers.insert(reqwest::header::IF_NONE_MATCH, v);
}
}
if let Some(ref date) = request.if_modified_since {
if let Ok(v) = HeaderValue::from_str(date) {
headers.insert(reqwest::header::IF_MODIFIED_SINCE, v);
}
}
headers
}
#[cfg(feature = "bot-auth")]
pub(crate) fn apply_bot_auth_if_enabled(
mut headers: HeaderMap,
options: &FetchOptions,
url: &Url,
) -> HeaderMap {
if let Some(ref bot_auth) = options.bot_auth {
if let Some(authority) = url.host_str() {
match bot_auth.sign_request(authority) {
Ok(auth_headers) => {
if let Ok(v) = HeaderValue::from_str(&auth_headers.signature) {
headers.insert("signature", v);
}
if let Ok(v) = HeaderValue::from_str(&auth_headers.signature_input) {
headers.insert("signature-input", v);
}
if let Some(ref fqdn) = auth_headers.signature_agent {
if let Ok(v) = HeaderValue::from_str(fqdn) {
headers.insert("signature-agent", v);
}
}
}
Err(e) => {
warn!("Bot-auth signing failed: {e}");
}
}
}
}
headers
}
#[cfg(not(feature = "bot-auth"))]
pub(crate) fn apply_bot_auth_if_enabled(
headers: HeaderMap,
_options: &FetchOptions,
_url: &Url,
) -> HeaderMap {
headers
}
struct ResponseMeta {
content_type: Option<String>,
last_modified: Option<String>,
etag: Option<String>,
content_length: Option<u64>,
filename: Option<String>,
}
fn extract_response_meta(headers: &[(String, String)], url: &str) -> ResponseMeta {
ResponseMeta {
content_type: header_value(headers, "content-type").map(|s| s.to_string()),
last_modified: header_value(headers, "last-modified").map(|s| s.to_string()),
etag: header_value(headers, "etag").map(|s| s.to_string()),
content_length: header_value(headers, "content-length").and_then(|s| s.parse().ok()),
filename: extract_filename(headers, url),
}
}
fn resources_from_link_headers(values: &[String], base_url: &str) -> Vec<AgentResource> {
let Ok(base_url) = Url::parse(base_url) else {
return Vec::new();
};
values
.iter()
.flat_map(|value| split_link_header(value))
.filter_map(|part| {
let (target, params) = part.split_once('>')?;
let target = target.trim().strip_prefix('<')?;
let relation = link_parameter(params, "rel");
let media_type = link_parameter(params, "type");
if !is_agent_link(relation.as_deref(), media_type.as_deref(), target) {
return None;
}
let url = base_url.join(target).ok()?;
Some(AgentResource {
kind: classify_agent_resource(
url.as_str(),
relation.as_deref().unwrap_or_default(),
media_type.as_deref(),
),
url: url.to_string(),
source: "http-link".to_string(),
relation,
media_type,
title: link_parameter(params, "title"),
verified: false,
})
})
.collect()
}
fn split_link_header(value: &str) -> Vec<&str> {
let mut parts = Vec::new();
let mut start = 0;
let mut quoted = false;
for (index, ch) in value.char_indices() {
match ch {
'"' => quoted = !quoted,
',' if !quoted => {
parts.push(value[start..index].trim());
start = index + 1;
}
_ => {}
}
}
parts.push(value[start..].trim());
parts
}
fn link_parameter(params: &str, name: &str) -> Option<String> {
params.split(';').skip(1).find_map(|part| {
let (key, value) = part.trim().split_once('=')?;
key.eq_ignore_ascii_case(name)
.then(|| value.trim().trim_matches('"').to_string())
})
}
fn is_agent_link(relation: Option<&str>, media_type: Option<&str>, target: &str) -> bool {
let relation = relation.unwrap_or_default().to_ascii_lowercase();
let media_type = media_type.unwrap_or_default().to_ascii_lowercase();
let target = target.to_ascii_lowercase();
let relevant_relation = relation.split_ascii_whitespace().any(|value| {
matches!(
value,
"alternate"
| "service-desc"
| "describedby"
| "authorization_endpoint"
| "mcp"
| "a2a"
| "agent-card"
| "skill"
)
});
relevant_relation
&& (media_type.contains("markdown")
|| media_type.contains("json")
|| target.contains("llms")
|| target.contains("auth.md")
|| target.contains("well-known")
|| relation != "alternate")
}
fn normalize_html_resources(resources: &mut Vec<AgentResource>, base_url: &str) {
let Ok(base_url) = Url::parse(base_url) else {
resources.clear();
return;
};
resources.retain_mut(|resource| match base_url.join(&resource.url) {
Ok(url) if matches!(url.scheme(), "http" | "https") => {
resource.url = url.to_string();
true
}
_ => false,
});
}
fn deduplicate_resources(resources: &mut Vec<AgentResource>) {
let mut seen = HashSet::new();
resources.retain(|resource| seen.insert(resource.url.clone()));
resources.truncate(20);
}
fn append_agent_resources(content: &mut String, resources: &[AgentResource]) {
content.push_str("\n\n---\n\n## Agent resources\n\n");
for resource in resources {
let label = resource.title.as_deref().unwrap_or(&resource.kind);
let status = if resource.verified {
"verified"
} else {
"advertised"
};
content.push_str(&format!(
"- [{label}]({}) — `{}`; {status} via {}\n",
resource.url, resource.kind, resource.source
));
}
}
fn probe_content_type_matches(kind: &str, content_type: Option<&str>) -> bool {
let Some(content_type) = content_type else {
return true;
};
let content_type = content_type.to_ascii_lowercase();
match kind {
"llms-txt" | "llms-full-txt" | "auth" => {
content_type.starts_with("text/plain") || content_type.contains("markdown")
}
_ => content_type.contains("json"),
}
}
async fn probe_agent_resources(base_url: &str, options: &FetchOptions) -> Vec<AgentResource> {
let Ok(base_url) = Url::parse(base_url) else {
return Vec::new();
};
let origin = base_url.origin().ascii_serialization();
let probes = AGENT_RESOURCE_PROBES
.iter()
.map(|(path, kind)| ((*path).to_string(), (*kind).to_string()))
.collect::<Vec<_>>();
futures::stream::iter(probes)
.map(|(path, kind)| {
let url = Url::parse(&format!("{origin}{path}"));
async move {
let url = url.ok()?;
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_static(DEFAULT_USER_AGENT));
let (response, redirects) = send_request_following_redirects(
url.clone(),
reqwest::Method::HEAD,
headers,
options,
AGENT_PROBE_TIMEOUT,
)
.await
.ok()?;
if !(200..300).contains(&response.status) || !redirects.is_empty() {
return None;
}
let meta = extract_response_meta(&response.headers, url.as_str());
if !probe_content_type_matches(&kind, meta.content_type.as_deref()) {
return None;
}
Some(AgentResource {
url: url.to_string(),
kind,
source: "probe".to_string(),
relation: None,
media_type: meta.content_type,
title: None,
verified: true,
})
}
})
.buffer_unordered(4)
.filter_map(|resource| async move { resource })
.collect()
.await
}
#[async_trait]
impl Fetcher for DefaultFetcher {
fn name(&self) -> &'static str {
"default"
}
fn matches(&self, _url: &Url) -> bool {
true
}
async fn fetch(
&self,
request: &FetchRequest,
options: &FetchOptions,
) -> Result<FetchResponse, FetchError> {
let request = request.normalized_for_fetch()?;
if request.url.is_empty() {
return Err(FetchError::MissingUrl);
}
let method = request.effective_method();
let wants_markdown = options.enable_markdown && request.wants_markdown();
let wants_text = options.enable_text && request.wants_text();
validate_rakers_render_request(&request, options)?;
let max_body_size = options.max_body_size.unwrap_or(DEFAULT_MAX_BODY_SIZE);
let accept = if wants_markdown {
"text/markdown, text/html;q=0.9, text/plain;q=0.8, */*;q=0.7"
} else if wants_text {
"text/html, text/plain, */*;q=0.8"
} else {
"*/*"
};
let headers = build_headers(options, accept, &request);
let parsed_url = url::Url::parse(&request.url).map_err(|_| FetchError::InvalidUrlScheme)?;
let reqwest_method = match method {
HttpMethod::Get => reqwest::Method::GET,
HttpMethod::Head => reqwest::Method::HEAD,
};
let (response, redirect_chain) = send_request_following_redirects(
parsed_url,
reqwest_method,
headers,
options,
FIRST_BYTE_TIMEOUT,
)
.await?;
let status_code = response.status;
let final_url = response.url.to_string();
let link_headers = response
.headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("link"))
.map(|(_, value)| value.clone())
.collect::<Vec<_>>();
let meta = extract_response_meta(&response.headers, &final_url);
if status_code == 304 {
return Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
last_modified: meta.last_modified,
etag: meta.etag,
..Default::default()
});
}
if method == HttpMethod::Head {
return Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: meta.content_length,
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
method: Some("HEAD".to_string()),
redirect_chain,
..Default::default()
});
}
let parsed_final_url = Url::parse(&final_url).map_err(|_| FetchError::InvalidUrlScheme)?;
let output_format = if wants_markdown {
ContentOutputFormat::Markdown
} else if wants_text {
ContentOutputFormat::Text
} else {
ContentOutputFormat::Raw
};
let metadata_processor = self.content_processors.find_for_output(
&parsed_final_url,
meta.content_type.as_deref(),
output_format,
);
if let Some(ref ct) = meta.content_type {
if metadata_processor.is_none() && is_binary_content_type(ct) {
let error = unsupported_binary_message(&parsed_final_url, ct);
return Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: meta.content_length,
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
redirect_chain,
error: Some(error),
quality: Some(binary_quality_signal()),
..Default::default()
});
}
}
let input_limit = metadata_processor
.and_then(|processor| processor.max_input_bytes())
.unwrap_or(max_body_size);
let (body, input_truncated) =
read_body_with_timeout(response, BODY_TIMEOUT, input_limit).await?;
let size = body.len() as u64;
let mut content = String::from_utf8_lossy(&body).to_string();
let is_html_content = is_html(&meta.content_type, &content);
let mut rendered_truncated = false;
let rendered_by = if is_html_content && request.wants_rakers_render() {
content = render_html_with_rakers(content, final_url.clone(), options).await?;
rendered_truncated = truncate_string_to_max_bytes(&mut content, max_body_size);
Some("rakers".to_string())
} else {
None
};
let truncated = input_truncated || rendered_truncated;
let is_paywall = detect_paywall(&content);
let content_processor = metadata_processor.or_else(|| {
if is_html_content {
self.content_processors.find_for_output(
&parsed_final_url,
Some("text/html"),
output_format,
)
} else {
None
}
});
if let Some(processor) = content_processor {
let processor_name = processor.name();
if truncated && !processor.accepts_truncated_input() {
let mut quality = binary_quality_signal();
quality.warnings.push("truncated".to_string());
return Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: Some(size),
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
truncated: Some(true),
redirect_chain,
error: Some(format!(
"Content processor {processor_name} cannot process a partial response body"
)),
quality: Some(quality),
rendered_by,
..Default::default()
});
}
debug!(processor = processor_name, "Processing response content");
let processor_body = if rendered_by.is_some() {
Bytes::from(content.clone())
} else {
body.clone()
};
let processed = processor
.process(ContentProcessorInput {
url: parsed_final_url,
content_type: meta.content_type.clone(),
body: processor_body,
output_format,
content_focus: ContentFocus::from_request(request.content_focus.as_deref()),
})
.await;
return match processed {
Ok(mut processed) => {
let output_truncated = processed
.content
.as_mut()
.map(|content| truncate_string_to_max_bytes(content, max_body_size))
.unwrap_or(false);
let truncated = truncated || output_truncated;
let mut final_content = processed.content.take();
let mut resources = resources_from_link_headers(&link_headers, &final_url);
if is_html_content {
if let Some(metadata) = &mut processed.metadata {
normalize_html_resources(&mut metadata.agent_resources, &final_url);
resources.append(&mut metadata.agent_resources);
}
if wants_markdown {
resources.extend(probe_agent_resources(&final_url, options).await);
}
}
deduplicate_resources(&mut resources);
if !resources.is_empty() {
if wants_markdown {
if let Some(content) = &mut final_content {
append_agent_resources(content, &resources);
}
}
processed
.metadata
.get_or_insert_with(Default::default)
.agent_resources = resources;
}
if truncated {
if let Some(content) = &mut final_content {
content.push_str(TRUNCATION_MESSAGE);
}
}
let word_count = final_content.as_deref().map(count_words);
let extraction_method = processed
.metadata
.as_ref()
.and_then(|metadata| metadata.extraction_method.as_deref());
let had_processor_quality = processed.quality.is_some();
let mut quality = processed.quality.unwrap_or_else(|| {
final_content
.as_deref()
.map_or_else(binary_quality_signal, |content| {
compute_quality_signal(
content,
status_code,
truncated,
is_paywall,
extraction_method,
word_count.unwrap_or_default(),
)
})
});
if truncated {
push_warning(&mut quality.warnings, "truncated");
if had_processor_quality {
quality.score = (quality.score - 0.2).max(0.0);
}
}
Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: Some(size),
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
format: processed.format,
content: final_content,
truncated: truncated.then_some(true),
metadata: processed.metadata,
quality: Some(quality),
word_count,
redirect_chain,
error: processed.error,
is_paywall: is_paywall.then_some(true),
rendered_by,
..Default::default()
})
}
Err(error) => Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: Some(size),
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
redirect_chain,
error: Some(format!(
"Content processor {processor_name} failed: {error}"
)),
quality: Some(binary_quality_signal()),
rendered_by,
..Default::default()
}),
};
}
let mut page_metadata: Option<PageMetadata> = None;
let (format, final_content, extraction_method) =
if is_markdown_content_type(&meta.content_type) && wants_markdown {
debug!("Content-type is markdown; skipping HTML conversion");
("markdown".to_string(), content, Some("native_markdown"))
} else if is_plain_text_content_type(&meta.content_type) && wants_text {
debug!("Content-type is plain text; skipping HTML conversion");
("text".to_string(), content, Some("native_text"))
} else if is_html_content {
("raw".to_string(), content, Some("full"))
} else {
("raw".to_string(), content, Some("raw"))
};
let mut final_content = filter_excessive_newlines(&final_content);
if truncated {
final_content.push_str(TRUNCATION_MESSAGE);
}
let mut resources = resources_from_link_headers(&link_headers, &final_url);
if let Some(metadata) = &mut page_metadata {
normalize_html_resources(&mut metadata.agent_resources, &final_url);
resources.append(&mut metadata.agent_resources);
}
if wants_markdown {
resources.extend(probe_agent_resources(&final_url, options).await);
}
deduplicate_resources(&mut resources);
if !resources.is_empty() {
if wants_markdown {
append_agent_resources(&mut final_content, &resources);
}
page_metadata
.get_or_insert_with(Default::default)
.agent_resources = resources;
}
let word_count = count_words(&final_content);
if let (Some(metadata), Some(method)) = (&mut page_metadata, extraction_method) {
metadata.extraction_method = Some(method.to_string());
}
let quality = compute_quality_signal(
&final_content,
status_code,
truncated,
is_paywall,
extraction_method,
word_count,
);
Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: Some(size),
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
format: Some(format),
content: Some(final_content),
truncated: if truncated { Some(true) } else { None },
metadata: page_metadata,
quality: Some(quality),
word_count: Some(word_count),
redirect_chain,
is_paywall: if is_paywall { Some(true) } else { None },
rendered_by,
..Default::default()
})
}
async fn fetch_to_file(
&self,
request: &FetchRequest,
options: &FetchOptions,
saver: &dyn FileSaver,
) -> Result<FetchResponse, FetchError> {
let save_path = match super::preflight_save_path(request, saver).await? {
Some(path) => path,
None => return self.fetch(request, options).await,
};
let request = request.normalized_for_fetch()?;
if request.url.is_empty() {
return Err(FetchError::MissingUrl);
}
let method = request.effective_method();
let max_body_size = options.max_body_size.unwrap_or(DEFAULT_MAX_BODY_SIZE);
let headers = build_headers(options, "*/*", &request);
let parsed_url = url::Url::parse(&request.url).map_err(|_| FetchError::InvalidUrlScheme)?;
let reqwest_method = match method {
HttpMethod::Get => reqwest::Method::GET,
HttpMethod::Head => reqwest::Method::HEAD,
};
let (response, redirect_chain) = send_request_following_redirects(
parsed_url,
reqwest_method,
headers,
options,
FIRST_BYTE_TIMEOUT,
)
.await?;
let status_code = response.status;
let final_url = response.url.to_string();
let meta = extract_response_meta(&response.headers, &final_url);
if method == HttpMethod::Head {
return Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: meta.content_length,
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
method: Some("HEAD".to_string()),
redirect_chain,
..Default::default()
});
}
let (body, truncated) =
read_body_with_timeout(response, BODY_TIMEOUT, max_body_size).await?;
let size = body.len() as u64;
let save_result = saver
.save(save_path, &body)
.await
.map_err(|e| FetchError::SaveError(e.to_string()))?;
Ok(FetchResponse {
url: final_url,
status_code,
content_type: meta.content_type,
size: Some(size),
last_modified: meta.last_modified,
etag: meta.etag,
filename: meta.filename,
truncated: if truncated { Some(true) } else { None },
saved_path: Some(save_result.path),
bytes_written: Some(save_result.bytes_written),
redirect_chain,
..Default::default()
})
}
}
pub(crate) async fn send_request_following_redirects(
initial_url: Url,
method: reqwest::Method,
headers: HeaderMap,
options: &FetchOptions,
timeout: Duration,
) -> Result<(TransportResponse, Vec<String>), FetchError> {
let transport = options.transport();
let transport_method = if method == reqwest::Method::HEAD {
TransportMethod::Head
} else {
TransportMethod::Get
};
let mut current_url = initial_url;
let mut redirect_chain = Vec::new();
for redirect_count in 0..=MAX_REDIRECTS {
super::validate_url_policy(¤t_url, options)?;
let request_headers = apply_bot_auth_if_enabled(headers.clone(), options, ¤t_url);
let pinned_addrs = pinned_addrs_for_url(¤t_url, options)?;
let req = TransportRequest {
method: transport_method,
url: current_url.clone(),
headers: headers_to_pairs(&request_headers),
timeout: Some(timeout),
pinned_addrs,
respect_proxy_env: options.respect_proxy_env,
};
let response = transport.execute(req).await?;
let Some(next_url) = redirect_target(¤t_url, &response, options)? else {
return Ok((response, redirect_chain));
};
if redirect_count == MAX_REDIRECTS {
return Err(FetchError::RequestError("too many redirects".to_string()));
}
debug!(
from = %current_url,
to = %next_url,
hop = redirect_count + 1,
"Following redirect with IP validation"
);
redirect_chain.push(current_url.to_string());
current_url = next_url;
}
unreachable!("redirect loop must return before exhausting iterations");
}
pub(crate) async fn transport_request(
url: Url,
method: reqwest::Method,
headers: HeaderMap,
options: &FetchOptions,
timeout: Duration,
pin_host: &str,
pin_port: u16,
) -> Result<TransportResponse, FetchError> {
super::validate_url_policy(&url, options)?;
let transport = options.transport();
let transport_method = if method == reqwest::Method::HEAD {
TransportMethod::Head
} else {
TransportMethod::Get
};
let request_headers = apply_bot_auth_if_enabled(headers, options, &url);
let pinned_addrs = options
.dns_policy
.pinned_addrs(pin_host, pin_port)
.map_err(|_| FetchError::BlockedUrl)?;
let req = TransportRequest {
method: transport_method,
url,
headers: headers_to_pairs(&request_headers),
timeout: Some(timeout),
pinned_addrs,
respect_proxy_env: options.respect_proxy_env,
};
Ok(transport.execute(req).await?)
}
pub(crate) async fn read_full_body(
response: TransportResponse,
options: &FetchOptions,
) -> Result<Bytes, FetchError> {
let max_body_size = options.max_body_size.unwrap_or(DEFAULT_MAX_BODY_SIZE);
let (body, _truncated) = read_body_with_timeout(response, BODY_TIMEOUT, max_body_size).await?;
Ok(body)
}
fn redirect_target(
base_url: &Url,
response: &TransportResponse,
options: &FetchOptions,
) -> Result<Option<Url>, FetchError> {
let status = response.status;
let is_redirection = (300..400).contains(&status);
if !is_redirection || status == 304 {
return Ok(None);
}
let location = header_value(&response.headers, LOCATION.as_str()).ok_or_else(|| {
FetchError::RequestError("redirect response missing Location header".to_string())
})?;
let next_url = base_url.join(location).map_err(|_| {
FetchError::RequestError("redirect Location is not a valid URL".to_string())
})?;
if next_url.scheme() != "http" && next_url.scheme() != "https" {
return Err(FetchError::InvalidUrlScheme);
}
options.validate_redirect_target(base_url, &next_url)?;
Ok(Some(next_url))
}
fn validate_rakers_render_request(
request: &FetchRequest,
options: &FetchOptions,
) -> Result<(), FetchError> {
if !request.wants_rakers_render() {
return Ok(());
}
if !options.enable_render_rakers {
return Err(FetchError::RenderNotAvailable);
}
#[cfg(feature = "render-rakers")]
{
Ok(())
}
#[cfg(not(feature = "render-rakers"))]
{
Err(FetchError::RenderNotAvailable)
}
}
fn truncate_string_to_max_bytes(content: &mut String, max_size: usize) -> bool {
if content.len() <= max_size {
return false;
}
let mut end = max_size;
while end > 0 && !content.is_char_boundary(end) {
end -= 1;
}
content.truncate(end);
true
}
#[cfg(feature = "render-rakers")]
async fn render_html_with_rakers(
html: String,
page_url: String,
options: &FetchOptions,
) -> Result<String, FetchError> {
let user_agent = options.user_agent.clone();
tokio::task::spawn_blocking(move || {
let deny_proxy = DenyProxy::new().map_err(|_| {
FetchError::RequestError(
"failed to initialize rendered-fetch network guard".to_string(),
)
})?;
let cfg = rakers::HttpConfig {
user_agent,
headers: Vec::new(),
proxy: Some(deny_proxy.url()),
forward_headers: false,
};
rakers::render(
&html,
false,
Some(&page_url),
&cfg,
true,
Some(0),
Some(RAKERS_SCRIPT_TIMEOUT),
)
.map_err(|err| FetchError::FetcherError(format!("rakers render failed: {err}")))
})
.await
.map_err(|_| FetchError::FetcherError("rakers render task failed".to_string()))?
}
#[cfg(not(feature = "render-rakers"))]
async fn render_html_with_rakers(
_html: String,
_page_url: String,
_options: &FetchOptions,
) -> Result<String, FetchError> {
Err(FetchError::RenderNotAvailable)
}
#[cfg(feature = "render-rakers")]
struct DenyProxy {
addr: std::net::SocketAddr,
stop: std::sync::Arc<std::sync::atomic::AtomicBool>,
handle: Option<std::thread::JoinHandle<()>>,
}
#[cfg(feature = "render-rakers")]
impl DenyProxy {
fn new() -> std::io::Result<Self> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
listener.set_nonblocking(true)?;
let addr = listener.local_addr()?;
let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let thread_stop = std::sync::Arc::clone(&stop);
let handle = std::thread::spawn(move || {
while !thread_stop.load(std::sync::atomic::Ordering::Relaxed) {
match listener.accept() {
Ok((mut stream, _)) => {
let _ = std::io::Write::write_all(
&mut stream,
b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
);
}
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
std::thread::sleep(Duration::from_millis(5));
}
Err(_) => break,
}
}
});
Ok(Self {
addr,
stop,
handle: Some(handle),
})
}
fn url(&self) -> String {
format!("http://{}", self.addr)
}
}
#[cfg(feature = "render-rakers")]
impl Drop for DenyProxy {
fn drop(&mut self) {
self.stop.store(true, std::sync::atomic::Ordering::Relaxed);
let _ = std::net::TcpStream::connect(self.addr);
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
}
const UNSUPPORTED_BINARY_MESSAGE: &str =
"Binary content is not supported. Only textual content (HTML, text, JSON, etc.) can be fetched.";
fn unsupported_binary_message(url: &Url, content_type: &str) -> String {
match DefaultFormatDetector.detect(url, Some(content_type)) {
Some(format) => format!(
"{UNSUPPORTED_BINARY_MESSAGE} {} conversion is not enabled in this fetchkit configuration; enable a document converter to convert it.",
format.as_str().to_ascii_uppercase()
),
None => UNSUPPORTED_BINARY_MESSAGE.to_string(),
}
}
fn is_binary_content_type(content_type: &str) -> bool {
let ct_lower = content_type.to_lowercase();
BINARY_PREFIXES
.iter()
.any(|prefix| ct_lower.starts_with(prefix))
}
fn extract_filename(headers: &[(String, String)], url: &str) -> Option<String> {
if let Some(value) = header_value(headers, CONTENT_DISPOSITION.as_str()) {
if let Some(filename) = parse_content_disposition_filename(value) {
return Some(filename);
}
}
if let Ok(parsed) = url::Url::parse(url) {
if let Some(mut segments) = parsed.path_segments() {
if let Some(last) = segments.next_back() {
if last.contains('.') && !last.is_empty() {
return Some(last.to_string());
}
}
}
}
None
}
fn parse_content_disposition_filename(value: &str) -> Option<String> {
let patterns = ["filename=\"", "filename="];
for pattern in patterns {
if let Some(start) = value.find(pattern) {
let rest = &value[start + pattern.len()..];
if pattern.ends_with('"') {
if let Some(end) = rest.find('"') {
return Some(rest[..end].to_string());
}
} else {
let end = rest
.find(|c: char| c.is_whitespace() || c == ';')
.unwrap_or(rest.len());
let filename = rest[..end].trim_matches('"');
if !filename.is_empty() {
return Some(filename.to_string());
}
}
}
}
None
}
pub(crate) async fn read_body_with_timeout(
response: TransportResponse,
timeout: Duration,
max_size: usize,
) -> Result<(Bytes, bool), FetchError> {
read_body_stream_with_timeout(response.body, timeout, max_size).await
}
pub(crate) async fn read_body_stream_with_timeout(
mut stream: BodyStream,
timeout: Duration,
max_size: usize,
) -> Result<(Bytes, bool), FetchError> {
let mut body = Vec::new();
let deadline = tokio::time::Instant::now() + timeout;
loop {
let chunk_future = stream.next();
let timeout_future = tokio::time::sleep_until(deadline);
tokio::select! {
chunk = chunk_future => {
match chunk {
Some(Ok(bytes)) => {
let remaining = max_size.saturating_sub(body.len());
if remaining == 0 {
warn!("Body size limit reached ({}), truncating", max_size);
return Ok((Bytes::from(body), true));
}
if bytes.len() > remaining {
body.extend_from_slice(&bytes[..remaining]);
warn!("Body size limit reached ({}), truncating", max_size);
return Ok((Bytes::from(body), true));
}
body.extend_from_slice(&bytes);
}
Some(Err(e)) => {
error!("Error reading body chunk: {}", e);
if body.is_empty() {
return Err(e.into());
}
return Ok((Bytes::from(body), true));
}
None => {
return Ok((Bytes::from(body), false));
}
}
}
_ = timeout_future => {
warn!("Body timeout reached, returning partial content");
return Ok((Bytes::from(body), true));
}
}
}
}
fn count_words(text: &str) -> u64 {
text.split_whitespace().count() as u64
}
fn binary_quality_signal() -> PageQuality {
PageQuality {
score: 0.0,
warnings: vec!["binary_content".to_string()],
suggested_next_action: Some("use_save_to_file".to_string()),
..Default::default()
}
}
fn compute_quality_signal(
content: &str,
status_code: u16,
truncated: bool,
is_paywall: bool,
extraction_method: Option<&str>,
word_count: u64,
) -> PageQuality {
let mut warnings = Vec::new();
let mut score = 1.0f32;
let link_count = count_markdown_links(content);
let link_density = if word_count == 0 {
0.0
} else {
link_count as f32 / word_count as f32
};
let lower = content.to_lowercase();
if status_code >= 400 {
push_warning(&mut warnings, "http_error");
score -= 0.35;
}
if truncated {
push_warning(&mut warnings, "truncated");
score -= 0.20;
}
if word_count < 30 {
push_warning(&mut warnings, "low_content");
score -= 0.30;
}
if link_count >= 20 && link_density > 0.15 {
push_warning(&mut warnings, "too_many_links");
score -= 0.20;
}
if is_paywall {
push_warning(&mut warnings, "possible_paywall");
score -= 0.25;
}
if looks_like_login_wall(&lower) {
push_warning(&mut warnings, "possible_login_wall");
score -= 0.25;
}
if looks_like_consent_wall(&lower) {
push_warning(&mut warnings, "possible_consent_wall");
score -= 0.20;
}
if looks_like_javascript_required(&lower) {
push_warning(&mut warnings, "javascript_required");
score -= 0.30;
}
PageQuality {
score: score.clamp(0.0, 1.0),
suggested_next_action: suggested_next_action(&warnings).map(str::to_string),
warnings,
link_density: Some(link_density),
extraction_method: extraction_method.map(str::to_string),
}
}
fn count_markdown_links(content: &str) -> usize {
content.matches("](").count()
}
fn push_warning(warnings: &mut Vec<String>, warning: &str) {
if !warnings.iter().any(|existing| existing == warning) {
warnings.push(warning.to_string());
}
}
fn suggested_next_action(warnings: &[String]) -> Option<&'static str> {
if has_warning(warnings, "javascript_required") {
Some("retry_with_browser_rendering")
} else if has_warning(warnings, "possible_login_wall") {
Some("authenticate_or_use_browser")
} else if has_warning(warnings, "possible_paywall") {
Some("try_alternate_source")
} else if has_warning(warnings, "truncated") {
Some("retry_with_larger_limit_or_narrower_scope")
} else if has_warning(warnings, "low_content") || has_warning(warnings, "too_many_links") {
Some("retry_with_agent_focus_or_crawl")
} else if has_warning(warnings, "http_error") {
Some("check_url_or_retry_later")
} else {
None
}
}
fn has_warning(warnings: &[String], warning: &str) -> bool {
warnings.iter().any(|existing| existing == warning)
}
fn looks_like_login_wall(lower_content: &str) -> bool {
[
"sign in to continue",
"log in to continue",
"please sign in",
"please log in",
"login required",
"sign in required",
]
.iter()
.any(|needle| lower_content.contains(needle))
}
fn looks_like_consent_wall(lower_content: &str) -> bool {
[
"accept cookies",
"cookie consent",
"manage cookies",
"privacy choices",
"we use cookies",
"consent preferences",
]
.iter()
.any(|needle| lower_content.contains(needle))
}
fn looks_like_javascript_required(lower_content: &str) -> bool {
[
"enable javascript",
"javascript is disabled",
"requires javascript",
"please enable js",
"enable js",
]
.iter()
.any(|needle| lower_content.contains(needle))
}
const PAYWALL_INDICATORS: &[&str] = &[
"paywall",
"subscribe to read",
"subscribe to continue",
"subscription required",
"premium content",
"members only",
"sign in to read",
"log in to read",
"create a free account",
"already a subscriber",
"unlock this article",
"get unlimited access",
"start your free trial",
];
fn detect_paywall(html: &str) -> bool {
let lower = html.to_lowercase();
PAYWALL_INDICATORS
.iter()
.any(|indicator| lower.contains(indicator))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::content::{ContentProcessor, ContentProcessorError, ProcessedContent};
use crate::dns::DnsPolicy;
use crate::types::FetchRequest;
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
struct BinaryIntegrityProcessor;
#[async_trait::async_trait]
impl ContentProcessor for BinaryIntegrityProcessor {
fn name(&self) -> &'static str {
"binary-integrity"
}
fn matches(&self, _url: &Url, content_type: Option<&str>) -> bool {
content_type == Some("application/pdf")
}
async fn process(
&self,
input: ContentProcessorInput,
) -> Result<ProcessedContent, ContentProcessorError> {
if input.body.as_ref() != b"%PDF-\xff\0binary" {
return Err(ContentProcessorError("binary bytes changed".to_string()));
}
Ok(ProcessedContent {
format: Some("markdown".to_string()),
content: Some("binary bytes intact".to_string()),
..Default::default()
})
}
}
#[test]
fn test_is_binary_content_type() {
assert!(is_binary_content_type("image/png"));
assert!(is_binary_content_type("image/jpeg"));
assert!(is_binary_content_type("audio/mp3"));
assert!(is_binary_content_type("video/mp4"));
assert!(is_binary_content_type("application/pdf"));
assert!(is_binary_content_type("application/x-pdf"));
assert!(is_binary_content_type("application/octet-stream"));
assert!(is_binary_content_type("application/zip"));
assert!(is_binary_content_type("application/vnd.ms-excel"));
assert!(is_binary_content_type(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
));
assert!(is_binary_content_type("font/woff2"));
assert!(!is_binary_content_type("text/html"));
assert!(!is_binary_content_type("text/plain"));
assert!(!is_binary_content_type("application/json"));
assert!(!is_binary_content_type("application/javascript"));
}
#[tokio::test]
async fn content_processors_receive_original_binary_bytes() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/document"))
.respond_with(
ResponseTemplate::new(200)
.set_body_bytes(b"%PDF-\xff\0binary".to_vec())
.insert_header("content-type", "application/pdf"),
)
.mount(&server)
.await;
let mut processors = ContentProcessorRegistry::new();
processors.register(Box::new(BinaryIntegrityProcessor));
let fetcher = DefaultFetcher::with_content_processors(processors);
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/document", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.content.as_deref(), Some("binary bytes intact"));
assert!(response.error.is_none(), "{:?}", response.error);
}
#[tokio::test]
async fn custom_binary_registry_retains_html_processing() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/article"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("<html><body><h1>Article</h1></body></html>")
.insert_header("content-type", "text/html"),
)
.mount(&server)
.await;
let mut processors = ContentProcessorRegistry::new();
processors.register(Box::new(BinaryIntegrityProcessor));
let fetcher = DefaultFetcher::with_content_processors(processors);
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/article", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.format.as_deref(), Some("markdown"));
assert!(response.content.as_deref().unwrap().contains("# Article"));
}
#[test]
fn test_parse_content_disposition_filename() {
assert_eq!(
parse_content_disposition_filename("attachment; filename=\"file.pdf\""),
Some("file.pdf".to_string())
);
assert_eq!(
parse_content_disposition_filename("attachment; filename=file.pdf"),
Some("file.pdf".to_string())
);
assert_eq!(
parse_content_disposition_filename("inline; filename=\"report.xlsx\"; size=1234"),
Some("report.xlsx".to_string())
);
assert_eq!(parse_content_disposition_filename("inline"), None);
}
#[test]
fn test_extract_filename_from_url() {
let headers: Vec<(String, String)> = Vec::new();
assert_eq!(
extract_filename(&headers, "https://example.com/path/to/file.pdf"),
Some("file.pdf".to_string())
);
assert_eq!(
extract_filename(&headers, "https://example.com/path/to/document"),
None
);
assert_eq!(extract_filename(&headers, "https://example.com/"), None);
}
#[test]
fn test_default_fetcher_matches_all() {
let fetcher = DefaultFetcher::new();
let url = Url::parse("https://example.com").unwrap();
assert!(fetcher.matches(&url));
let url = Url::parse("https://github.com/owner/repo").unwrap();
assert!(fetcher.matches(&url));
}
#[tokio::test]
async fn test_manual_redirect_following() {
let destination = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/final"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("redirected")
.insert_header("content-type", "text/plain"),
)
.mount(&destination)
.await;
let origin = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/start"))
.respond_with(
ResponseTemplate::new(302)
.insert_header("location", format!("{}/final", destination.uri())),
)
.mount(&origin)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
enable_text: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/start", origin.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 200);
assert_eq!(response.content.as_deref(), Some("redirected"));
}
#[tokio::test]
async fn test_redirect_hop_honors_url_prefix_policy() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/docs/start"))
.respond_with(
ResponseTemplate::new(302)
.insert_header("location", format!("{}/private", server.uri())),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/private"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
allow_prefixes: vec![format!("{}/docs", server.uri())],
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/docs/start", server.uri()));
let result = fetcher.fetch(&request, &options).await;
assert!(matches!(result, Err(FetchError::BlockedUrl)));
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].url.path(), "/docs/start");
assert!(!requests
.iter()
.any(|request| request.url.path() == "/private"));
}
fn redirect_response(status: u16, location: &str) -> TransportResponse {
TransportResponse {
status,
url: Url::parse("https://origin.example/start").unwrap(),
headers: vec![("location".to_string(), location.to_string())],
body: Box::pin(futures::stream::empty()),
}
}
#[test]
fn test_redirect_target_handles_relative_location() {
let base_url = Url::parse("https://origin.example/start").unwrap();
let response = redirect_response(302, "/final");
let redirect = redirect_target(&base_url, &response, &FetchOptions::default()).unwrap();
assert_eq!(
redirect.unwrap(),
Url::parse("https://origin.example/final").unwrap()
);
}
#[test]
fn test_redirect_target_rejects_non_http_location() {
let base_url = Url::parse("https://origin.example/start").unwrap();
let response = redirect_response(302, "file:///etc/passwd");
let redirect = redirect_target(&base_url, &response, &FetchOptions::default());
assert!(matches!(redirect, Err(FetchError::InvalidUrlScheme)));
}
#[tokio::test]
async fn test_skip_conversion_for_markdown_content_type() {
let server = MockServer::start().await;
let md_body = "# Already Markdown\n\nThis is **already** formatted.";
Mock::given(method("GET"))
.and(path("/doc"))
.respond_with(
ResponseTemplate::new(200).set_body_raw(md_body, "text/markdown; charset=utf-8"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/doc", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.format.as_deref(), Some("markdown"));
assert!(response
.content
.as_deref()
.unwrap()
.contains("# Already Markdown"));
assert!(response.content.as_deref().unwrap().contains("**already**"));
}
#[tokio::test]
async fn test_skip_conversion_for_plain_text_content_type() {
let server = MockServer::start().await;
let text_body = "Just plain text\nwith newlines.";
Mock::given(method("GET"))
.and(path("/plain"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(text_body)
.insert_header("content-type", "text/plain"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_text: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/plain", server.uri())).as_text();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.format.as_deref(), Some("text"));
assert!(response
.content
.as_deref()
.unwrap()
.contains("Just plain text"));
}
#[tokio::test]
async fn test_agent_content_focus_prefers_readable_body() {
let server = MockServer::start().await;
let html = r#"
<html>
<body>
<nav><a href="/home">Home</a><a href="/pricing">Pricing</a></nav>
<article>
<h1>Agent Ready Article</h1>
<p>This is the useful content an autonomous AI agent should receive.</p>
<p>The second paragraph gives the readability scorer a clear signal.</p>
</article>
<aside>Related stories and subscription widgets</aside>
</body>
</html>
"#;
Mock::given(method("GET"))
.and(path("/article"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(html)
.insert_header("content-type", "text/html"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/article", server.uri()))
.as_markdown()
.content_focus("agent");
let response = fetcher.fetch(&request, &options).await.unwrap();
let content = response.content.as_deref().unwrap();
assert!(content.contains("# Agent Ready Article"), "{content}");
assert!(content.contains("useful content"), "{content}");
assert!(!content.contains("Pricing"), "{content}");
assert!(!content.contains("subscription widgets"), "{content}");
assert_eq!(
response
.metadata
.as_ref()
.and_then(|meta| meta.extraction_method.as_deref()),
Some("agent_readable")
);
}
#[tokio::test]
async fn test_markdown_resolves_relative_links_against_final_url() {
let server = MockServer::start().await;
let html = r#"
<html>
<body>
<main>
<p>Read <a href="/docs/api">API docs</a>.</p>
<img src="../assets/logo.png" alt="Logo">
</main>
</body>
</html>
"#;
Mock::given(method("GET"))
.and(path("/guide/start"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(html)
.insert_header("content-type", "text/html"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/guide/start", server.uri()))
.as_markdown()
.content_focus("main");
let response = fetcher.fetch(&request, &options).await.unwrap();
let content = response.content.as_deref().unwrap();
assert!(
content.contains(&format!("[API docs]({}/docs/api)", server.uri())),
"{content}"
);
assert!(
content.contains(&format!("", server.uri())),
"{content}"
);
}
#[tokio::test]
async fn test_markdown_content_type_without_markdown_request_returns_raw() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/doc"))
.respond_with(
ResponseTemplate::new(200).set_body_raw("# Title", "text/markdown; charset=utf-8"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/doc", server.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.format.as_deref(), Some("raw"));
assert!(response.content.as_deref().unwrap().contains("# Title"));
}
#[tokio::test]
async fn test_plain_text_content_type_without_text_request_returns_raw() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/plain"))
.respond_with(ResponseTemplate::new(200).set_body_raw("hello world", "text/plain"))
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/plain", server.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.format.as_deref(), Some("raw"));
}
#[cfg(feature = "bot-auth")]
#[tokio::test]
async fn test_bot_auth_headers_sent() {
use crate::bot_auth::BotAuthConfig;
use wiremock::matchers::header_exists;
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/authed"))
.and(header_exists("signature"))
.and(header_exists("signature-input"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("ok")
.insert_header("content-type", "text/plain"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
bot_auth: Some(BotAuthConfig::from_seed([10u8; 32])),
..Default::default()
};
let request = FetchRequest::new(format!("{}/authed", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 200);
assert_eq!(response.content.as_deref(), Some("ok"));
}
#[cfg(feature = "bot-auth")]
#[tokio::test]
async fn test_bot_auth_signature_agent_header_sent() {
use crate::bot_auth::BotAuthConfig;
use wiremock::matchers::{header, header_exists};
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/agent"))
.and(header_exists("signature"))
.and(header_exists("signature-input"))
.and(header("signature-agent", "bot.example.com"))
.respond_with(ResponseTemplate::new(200).set_body_string("agent ok"))
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
bot_auth: Some(BotAuthConfig::from_seed([11u8; 32]).with_agent_fqdn("bot.example.com")),
..Default::default()
};
let request = FetchRequest::new(format!("{}/agent", server.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 200);
}
#[tokio::test]
async fn test_etag_returned_in_response() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/page"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("content")
.insert_header("content-type", "text/plain")
.insert_header("etag", "\"abc123\""),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/page", server.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 200);
assert_eq!(response.etag.as_deref(), Some("\"abc123\""));
}
#[tokio::test]
async fn test_conditional_fetch_304_not_modified() {
use wiremock::matchers::header;
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/page"))
.and(header("if-none-match", "\"abc123\""))
.respond_with(ResponseTemplate::new(304).insert_header("etag", "\"abc123\""))
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request =
FetchRequest::new(format!("{}/page", server.uri())).if_none_match("\"abc123\"");
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 304);
assert_eq!(response.etag.as_deref(), Some("\"abc123\""));
assert!(response.content.is_none());
assert!(response.format.is_none());
}
#[tokio::test]
async fn test_conditional_fetch_if_modified_since() {
use wiremock::matchers::header_exists;
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/page"))
.and(header_exists("if-modified-since"))
.respond_with(ResponseTemplate::new(304))
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/page", server.uri()))
.if_modified_since("Wed, 21 Oct 2015 07:28:00 GMT");
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 304);
assert!(response.content.is_none());
}
#[test]
fn test_count_words() {
assert_eq!(count_words("hello world"), 2);
assert_eq!(count_words(""), 0);
assert_eq!(count_words(" one two three "), 3);
assert_eq!(count_words("word"), 1);
}
#[test]
fn test_compute_quality_signal_clean_content() {
let content = "This page has enough useful words for an AI agent to answer with confidence. It includes actual content instead of just a menu, and it gives a short but complete explanation that should be useful for downstream reasoning.";
let quality = compute_quality_signal(
content,
200,
false,
false,
Some("agent_readable"),
count_words(content),
);
assert!(quality.score > 0.9, "{quality:?}");
assert!(quality.warnings.is_empty(), "{quality:?}");
assert_eq!(quality.extraction_method.as_deref(), Some("agent_readable"));
assert!(quality.suggested_next_action.is_none());
}
#[test]
fn test_compute_quality_signal_low_js_content() {
let quality = compute_quality_signal(
"Please enable JavaScript to view this app.",
200,
false,
false,
Some("full"),
7,
);
assert!(quality.score < 0.5, "{quality:?}");
assert!(quality.warnings.contains(&"low_content".to_string()));
assert!(quality
.warnings
.contains(&"javascript_required".to_string()));
assert_eq!(
quality.suggested_next_action.as_deref(),
Some("retry_with_browser_rendering")
);
}
#[test]
fn test_detect_paywall() {
assert!(detect_paywall("<div class=\"paywall\">Subscribe</div>"));
assert!(detect_paywall("<p>Subscribe to read the full article</p>"));
assert!(detect_paywall("<span>Already a subscriber? Log in</span>"));
assert!(detect_paywall("<div>Unlock this article</div>"));
assert!(!detect_paywall("<p>This is a normal article</p>"));
assert!(!detect_paywall("<h1>Hello World</h1><p>Free content</p>"));
}
#[tokio::test]
async fn test_word_count_in_response() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/article"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("Hello world this is a test")
.insert_header("content-type", "text/plain"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/article", server.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.word_count, Some(6));
}
#[tokio::test]
async fn test_redirect_chain_tracked() {
let destination = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/final"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("arrived")
.insert_header("content-type", "text/plain"),
)
.mount(&destination)
.await;
let origin = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/start"))
.respond_with(
ResponseTemplate::new(302)
.insert_header("location", format!("{}/final", destination.uri())),
)
.mount(&origin)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/start", origin.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.status_code, 200);
assert_eq!(response.redirect_chain.len(), 1);
assert!(response.redirect_chain[0].contains("/start"));
}
#[tokio::test]
async fn test_no_redirect_chain_for_direct_response() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/direct"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("direct")
.insert_header("content-type", "text/plain"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/direct", server.uri()));
let response = fetcher.fetch(&request, &options).await.unwrap();
assert!(response.redirect_chain.is_empty());
}
#[test]
fn test_parse_agent_resources_from_link_headers() {
let resources = resources_from_link_headers(
&[r#"</llms.txt>; rel="alternate"; type="text/markdown"; title="LLM index", </style.css>; rel="stylesheet""#.to_string()],
"https://example.com/docs/page",
);
assert_eq!(resources.len(), 1);
assert_eq!(resources[0].url, "https://example.com/llms.txt");
assert_eq!(resources[0].kind, "llms-txt");
assert_eq!(resources[0].source, "http-link");
assert_eq!(resources[0].title.as_deref(), Some("LLM index"));
assert!(!resources[0].verified);
}
#[tokio::test]
async fn test_agent_resources_are_probed_and_appended_to_markdown() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/page"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(r#"<html><head><link rel="service-desc" href="/openapi.json" type="application/openapi+json"></head><body><p>Page</p></body></html>"#)
.insert_header("content-type", "text/html")
.insert_header("link", r#"</llms.txt>; rel="alternate"; type="text/markdown""#),
)
.mount(&server)
.await;
Mock::given(method("HEAD"))
.and(path("/llms.txt"))
.respond_with(ResponseTemplate::new(200).insert_header("content-type", "text/markdown"))
.mount(&server)
.await;
Mock::given(method("HEAD"))
.and(path("/auth.md"))
.respond_with(ResponseTemplate::new(200).insert_header("content-type", "text/markdown"))
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/page", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
let resources = &response.metadata.as_ref().unwrap().agent_resources;
assert_eq!(
resources
.iter()
.filter(|resource| resource.kind == "llms-txt")
.count(),
1
);
assert!(resources
.iter()
.any(|resource| resource.kind == "auth" && resource.verified));
assert!(resources
.iter()
.any(|resource| resource.kind == "api-description"));
let content = response.content.unwrap();
assert!(content.contains("## Agent resources"));
assert!(content.contains("/auth.md"));
assert!(content.contains("/openapi.json"));
}
#[tokio::test]
async fn test_agent_resource_probes_honor_url_prefix_policy() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/docs/page"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("<html><body><p>Page</p></body></html>")
.insert_header("content-type", "text/html"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
allow_prefixes: vec![format!("{}/docs", server.uri())],
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/docs/page", server.uri())).as_markdown();
fetcher.fetch(&request, &options).await.unwrap();
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].method, reqwest::Method::GET);
assert_eq!(requests[0].url.path(), "/docs/page");
}
#[tokio::test]
async fn test_paywall_detection() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/paywalled"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("<html><body><div class='paywall'>Subscribe to read the full article</div><p>Preview...</p></body></html>")
.insert_header("content-type", "text/html"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/paywalled", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert_eq!(response.is_paywall, Some(true));
}
#[tokio::test]
async fn test_no_paywall_for_normal_content() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/free"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string("<html><body><p>This is free content</p></body></html>")
.insert_header("content-type", "text/html"),
)
.mount(&server)
.await;
let fetcher = DefaultFetcher::new();
let options = FetchOptions {
enable_markdown: true,
dns_policy: DnsPolicy::allow_all(),
..Default::default()
};
let request = FetchRequest::new(format!("{}/free", server.uri())).as_markdown();
let response = fetcher.fetch(&request, &options).await.unwrap();
assert!(response.is_paywall.is_none());
}
}