mcp_servers/coding/tools/web_fetch/
mod.rs

1mod http_client;
2
3pub use http_client::{
4    DEFAULT_TIMEOUT_MS, HttpClient, HttpResponse, MAX_TIMEOUT_MS, ReqwestClient, WebFetchInput, WebFetchOutput,
5};
6
7use dom_smoothie::Readability;
8use htmd::convert;
9use reqwest::Url;
10use std::time::Duration;
11
12use crate::coding::error::WebFetchError;
13
14/// HTTP client for fetching web content and converting to markdown.
15pub struct WebFetcher {
16    client: Box<dyn HttpClient>,
17}
18
19impl Default for WebFetcher {
20    fn default() -> Self {
21        Self::new()
22    }
23}
24
25impl WebFetcher {
26    /// Creates a new `WebFetcher` with a preconfigured reqwest client
27    pub fn new() -> Self {
28        Self::with_client(ReqwestClient::new())
29    }
30
31    /// Creates a `WebFetcher` with a custom HTTP client (useful for testing)
32    pub fn with_client(client: impl HttpClient + 'static) -> Self {
33        Self { client: Box::new(client) }
34    }
35
36    /// Fetches web content and converts it to markdown
37    pub async fn fetch(&self, args: WebFetchInput) -> Result<WebFetchOutput, WebFetchError> {
38        let url = normalize_url(&args.url)?;
39
40        let timeout_ms = args.timeout.map_or(DEFAULT_TIMEOUT_MS, |t| t.min(MAX_TIMEOUT_MS));
41
42        let response = self.client.fetch(&url, Duration::from_millis(timeout_ms)).await?;
43
44        let is_html = response.content_type.as_deref().is_none_or(|content_type| {
45            let mime = content_type.split(';').next().unwrap_or_default().trim();
46            mime.eq_ignore_ascii_case("text/html") || mime.eq_ignore_ascii_case("application/xhtml+xml")
47        });
48        let extracted = if is_html {
49            extract_content(&response.body, &response.final_url)
50        } else {
51            ExtractedContent { title: None, markdown: response.body, byline: None }
52        };
53
54        Ok(WebFetchOutput {
55            content: extracted.markdown,
56            final_url: response.final_url,
57            status_code: response.status_code,
58            title: extracted.title,
59            byline: extracted.byline,
60        })
61    }
62}
63
64struct ExtractedContent {
65    title: Option<String>,
66    markdown: String,
67    byline: Option<String>,
68}
69
70fn extract_content(html: &str, url: &str) -> ExtractedContent {
71    let article = Readability::new(html, Some(url), None)
72        .ok()
73        .and_then(|mut reader| reader.parse().ok())
74        .filter(|article| !article.content.is_empty());
75
76    match article {
77        Some(article) => ExtractedContent {
78            markdown: convert(&article.content).unwrap_or_else(|_| article.text_content.to_string()),
79            title: non_empty(article.title),
80            byline: article.byline,
81        },
82        None => ExtractedContent {
83            title: extract_title(html),
84            markdown: convert(html).unwrap_or_else(|_| html.to_string()),
85            byline: None,
86        },
87    }
88}
89
90fn normalize_url(url: &str) -> Result<String, WebFetchError> {
91    let url = if url.starts_with("http://") {
92        url.replacen("http://", "https://", 1)
93    } else if !url.starts_with("https://") {
94        format!("https://{url}")
95    } else {
96        url.to_string()
97    };
98
99    Url::parse(&url).map(|u| u.to_string()).map_err(WebFetchError::InvalidUrl)
100}
101
102fn extract_title(html: &str) -> Option<String> {
103    let lower = html.to_lowercase();
104    let start = lower.find("<title>")?;
105    let end = lower.find("</title>")?;
106    if start >= end {
107        return None;
108    }
109    non_empty(html[start + 7..end].trim().to_string())
110}
111
112fn non_empty(s: String) -> Option<String> {
113    if s.trim().is_empty() { None } else { Some(s) }
114}
115
116#[cfg(test)]
117mod tests {
118    use super::*;
119
120    #[test]
121    fn test_normalize_url() {
122        let cases = [
123            ("example.com", "https://example.com/"),
124            ("http://example.com", "https://example.com/"),
125            ("https://example.com", "https://example.com/"),
126        ];
127        for (input, expected) in cases {
128            assert_eq!(normalize_url(input).unwrap(), expected, "input: {input}");
129        }
130        assert!(normalize_url("not a valid url!!!").is_err());
131    }
132
133    #[test]
134    fn test_extract_title() {
135        assert_eq!(extract_title("<html><head><title>Test Page</title></head></html>"), Some("Test Page".to_string()));
136        assert_eq!(extract_title("<html><head></head><body></body></html>"), None);
137        assert_eq!(extract_title("<html><head><title>  </title></head></html>"), None);
138    }
139
140    fn article_html(body: &str) -> String {
141        format!(
142            r#"<html><head><title>Test Article</title>
143            <meta name="author" content="Jane Doe">
144            <meta property="og:description" content="A test excerpt">
145            </head><body>
146            <nav><a href="/">Home</a><a href="/about">About</a></nav>
147            <article>{body}</article>
148            <footer><p>Copyright 2026</p></footer>
149            </body></html>"#
150        )
151    }
152
153    #[test]
154    fn test_readability_extracts_article_content() {
155        let body = "<h1>Main Article</h1>";
156        let paragraphs: Vec<String> = (0..10)
157            .map(|i| format!("<p>This is paragraph {i} with enough content to be considered substantial by the readability algorithm.</p>"))
158            .collect();
159        let html = article_html(&format!("{body}{}", paragraphs.join("")));
160
161        let result = extract_content(&html, "https://example.com/article");
162
163        assert!(result.markdown.contains("Main Article"));
164        assert!(result.markdown.contains("paragraph 0"));
165        assert!(!result.markdown.contains("Copyright 2026"), "Footer should be stripped by readability");
166    }
167
168    #[test]
169    fn test_readability_extracts_metadata() {
170        let paragraphs: Vec<String> =
171            (0..10).map(|i| format!("<p>Paragraph {i} with enough content for readability to engage.</p>")).collect();
172        let html = article_html(&paragraphs.join(""));
173
174        let result = extract_content(&html, "https://example.com/article");
175
176        assert!(result.title.is_some());
177    }
178
179    #[test]
180    fn test_fallback_when_no_article_content() {
181        let html = "<html><head><title>Sparse</title></head><body><p>Hi</p></body></html>";
182        let result = extract_content(html, "https://example.com/sparse");
183
184        assert_eq!(result.title, Some("Sparse".to_string()));
185        assert!(result.markdown.contains("Hi"));
186    }
187}
mcp_servers/coding/tools/web_fetch/mod.rs

mcp_servers/coding/tools/web_fetch/
mod.rs
