use std::time::Duration;
use url::Url;
pub const DEFAULT_USER_AGENT_AUTONOMOUS: &str =
"ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)";
pub const DEFAULT_USER_AGENT_MANUAL: &str =
"ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)";
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
pub const MAX_RESPONSE_BODY_BYTES: usize = 8 * 1024 * 1024;
fn install_rustls_provider() {
let _ = rustls::crypto::ring::default_provider().install_default();
}
pub struct FetchClient {
client: reqwest::Client,
}
impl FetchClient {
pub fn new(proxy_url: Option<&str>) -> Result<Self, String> {
install_rustls_provider();
let mut builder = reqwest::Client::builder();
if let Some(proxy) = proxy_url {
let proxy = reqwest::Proxy::all(proxy)
.map_err(|e| format!("Invalid proxy URL {proxy:?}: {e}"))?;
builder = builder.proxy(proxy);
}
let client = builder
.build()
.map_err(|e| format!("Failed to build HTTP client: {e}"))?;
Ok(Self { client })
}
pub async fn get(&self, url: &str, user_agent: &str) -> Result<(String, String, u16), String> {
let response = self
.client
.get(url)
.header(reqwest::header::USER_AGENT, user_agent)
.timeout(REQUEST_TIMEOUT)
.send()
.await
.map_err(|e| format!("Failed to fetch {url}: {e:?}"))?;
read_bounded_response(response, url).await
}
}
async fn read_bounded_response(
mut response: reqwest::Response,
url: &str,
) -> Result<(String, String, u16), String> {
let status = response.status().as_u16();
let content_type = response
.headers()
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or_default()
.to_string();
if response
.content_length()
.is_some_and(|length| length > MAX_RESPONSE_BODY_BYTES as u64)
{
return Err(response_too_large(url));
}
let mut body = Vec::with_capacity(
response
.content_length()
.map(|length| length.min(MAX_RESPONSE_BODY_BYTES as u64) as usize)
.unwrap_or(0),
);
loop {
let chunk = response
.chunk()
.await
.map_err(|e| format!("Failed to read response body from {url}: {e}"))?;
let Some(chunk) = chunk else {
break;
};
if body.len().saturating_add(chunk.len()) > MAX_RESPONSE_BODY_BYTES {
return Err(response_too_large(url));
}
body.extend_from_slice(&chunk);
}
Ok((decode_text(&body), content_type, status))
}
fn response_too_large(url: &str) -> String {
format!("Response body from {url} exceeds the {MAX_RESPONSE_BODY_BYTES} byte safety limit")
}
fn decode_text(body: &[u8]) -> String {
let body = body.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(body);
String::from_utf8_lossy(body).into_owned()
}
pub fn robots_txt_url(url: &str) -> Result<String, String> {
let parsed = Url::parse(url).map_err(|e| format!("Invalid URL {url}: {e}"))?;
Ok(format!(
"{}://{}/robots.txt",
parsed.scheme(),
parsed.authority()
))
}
pub async fn check_may_fetch(
client: &FetchClient,
url: &str,
user_agent: &str,
) -> Result<(), String> {
let robots_url = robots_txt_url(url)?;
let (body, _content_type, status) = client.get(&robots_url, user_agent).await.map_err(|e| {
format!("Failed to fetch robots.txt {robots_url} due to a connection issue: {e}")
})?;
if status == 401 || status == 403 {
return Err(format!(
"When fetching robots.txt ({robots_url}), received status {status} so assuming \
that autonomous fetching is not allowed, the user can try manually fetching \
by using the fetch prompt"
));
}
if (400..500).contains(&status) {
return Ok(());
}
let processed: String = body
.lines()
.filter(|line| !line.trim_start().starts_with('#'))
.collect::<Vec<_>>()
.join("\n");
let mut matcher = robotstxt::DefaultMatcher::default();
if !matcher.one_agent_allowed_by_robots(&processed, user_agent, url) {
return Err(format!(
"The sites robots.txt ({robots_url}), specifies that autonomous fetching of \
this page is not allowed, \
<useragent>{user_agent}</useragent>\n\
<url>{url}</url>\n\
<robots>\n{body}\n</robots>\n\
The assistant must let the user know that it failed to view the page. \
The assistant may provide further guidance based on the above information. \
The assistant can tell the user that they can try manually fetching the page \
by using the fetch prompt within their UI."
));
}
Ok(())
}
pub async fn fetch_url(
client: &FetchClient,
url: &str,
user_agent: &str,
force_raw: bool,
) -> Result<(String, String), String> {
let (page_raw, content_type, status) = client.get(url, user_agent).await?;
if status >= 400 {
return Err(format!("Failed to fetch {url} - status code {status}"));
}
let first_100: String = page_raw.chars().take(100).collect();
let is_page_html = first_100.contains("<html")
|| content_type.contains("text/html")
|| content_type.is_empty();
if is_page_html && !force_raw {
Ok((extract_content_from_html(&page_raw), String::new()))
} else {
Ok((
page_raw,
format!(
"Content type {content_type} cannot be simplified to markdown, but here is \
the raw content:\n"
),
))
}
}
pub fn extract_content_from_html(html: &str) -> String {
let content = parse_html(html);
let content = content.trim();
if content.is_empty() {
"<error>Page failed to be simplified from HTML</error>".to_string()
} else {
content.to_string()
}
}
fn parse_html(html: &str) -> String {
let mut custom: std::collections::HashMap<String, Box<dyn html2md::TagHandlerFactory>> =
std::collections::HashMap::new();
for tag in ["style", "script", "noscript", "template"] {
custom.insert(tag.to_string(), Box::new(SuppressFactory));
}
html2md::parse_html_custom(html, &custom)
}
#[derive(Default)]
struct SuppressHandler;
impl html2md::TagHandler for SuppressHandler {
fn handle(&mut self, _tag: &html2md::Handle, _printer: &mut html2md::StructuredPrinter) {}
fn after_handle(&mut self, _printer: &mut html2md::StructuredPrinter) {}
fn skip_descendants(&self) -> bool {
true
}
}
#[derive(Default)]
struct SuppressFactory;
impl html2md::TagHandlerFactory for SuppressFactory {
fn instantiate(&self) -> Box<dyn html2md::TagHandler> {
Box::new(SuppressHandler)
}
}
pub fn truncate(content: &str, start_index: usize, max_length: usize) -> String {
let chars: Vec<char> = content.chars().collect();
if start_index >= chars.len() {
return "<error>No more content available.</error>".to_string();
}
let end = (start_index + max_length).min(chars.len());
let actual = end - start_index;
let mut out: String = chars[start_index..end].iter().collect();
if actual == max_length && end < chars.len() {
let next_start = start_index + actual;
out.push_str(&format!(
"\n\n<error>Content truncated. Call the fetch tool with a start_index of \
{next_start} to get more content.</error>"
));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn robots_url_construction() {
assert_eq!(
robots_txt_url("https://example.com/page").unwrap(),
"https://example.com/robots.txt"
);
assert_eq!(
robots_txt_url("https://example.com/some/deep/path/page.html").unwrap(),
"https://example.com/robots.txt"
);
assert_eq!(
robots_txt_url("https://example.com/page?foo=bar&baz=qux").unwrap(),
"https://example.com/robots.txt"
);
assert_eq!(
robots_txt_url("https://example.com:8080/page").unwrap(),
"https://example.com:8080/robots.txt"
);
assert_eq!(
robots_txt_url("https://example.com/page#section").unwrap(),
"https://example.com/robots.txt"
);
assert_eq!(
robots_txt_url("http://example.com/page").unwrap(),
"http://example.com/robots.txt"
);
}
#[test]
fn robots_url_rejects_garbage() {
assert!(robots_txt_url("not a url").is_err());
}
#[test]
fn extract_simple_html() {
let html = "<html><body><article><h1>Hello World</h1><p>This is a test paragraph.</p></article></body></html>";
let result = extract_content_from_html(html);
assert!(result.contains("Hello World"), "got: {result}");
assert!(result.contains("test paragraph"), "got: {result}");
}
#[test]
fn extract_html_with_link() {
let html = "<html><body><article><p>Visit <a href=\"https://example.com\">Example</a> for more.</p></article></body></html>";
let result = extract_content_from_html(html);
assert!(result.contains("Example"), "got: {result}");
}
#[test]
fn extract_empty_returns_error() {
let result = extract_content_from_html("");
assert!(result.contains("<error>"));
}
#[test]
fn extract_drops_style_and_script_noise() {
let html = "<html><head><style>body{color:red;background:#eee}</style>\
<script>console.log('x')</script></head>\
<body><article><h1>Real Title</h1><p>Real content.</p></article></body></html>";
let result = extract_content_from_html(html);
assert!(result.contains("Real Title"), "got: {result}");
assert!(result.contains("Real content"), "got: {result}");
assert!(!result.contains("color:red"), "style leaked: {result}");
assert!(!result.contains("console.log"), "script leaked: {result}");
}
#[test]
fn truncate_basic_slice() {
assert_eq!(truncate("abc", 0, 3), "abc");
let result = truncate("abcdef", 0, 3);
assert!(result.starts_with("abc"));
assert!(result.contains("start_index of 3"));
let result = truncate("abcdef", 2, 2);
assert!(result.starts_with("cd"));
assert!(result.contains("start_index of 4"));
}
#[test]
fn truncate_with_continuation_hint() {
let result = truncate("abcdefgh", 0, 5);
assert!(result.starts_with("abcde"));
assert!(result.contains("Content truncated"));
assert!(result.contains("start_index of 5"));
}
#[test]
fn truncate_exact_boundary_has_no_hint() {
let result = truncate("abcdef", 0, 6);
assert_eq!(result, "abcdef");
}
#[test]
fn truncate_past_end_returns_error() {
assert_eq!(
truncate("abcdef", 6, 5),
"<error>No more content available.</error>"
);
assert_eq!(
truncate("abcdef", 100, 5),
"<error>No more content available.</error>"
);
}
#[test]
fn truncate_is_character_indexed() {
let result = truncate("héllo wörld", 0, 5);
assert!(result.starts_with("héllo"));
assert!(result.contains("start_index of 5"));
let result = truncate("héllo wörld", 6, 5);
assert_eq!(result, "wörld");
}
}