mcp_servers/coding/tools/web_fetch/
http_client.rs

1use schemars::JsonSchema;
2use serde::{Deserialize, Serialize};
3use std::future::Future;
4use std::time::Duration;
5
6use crate::coding::error::WebFetchError;
7
8pub const DEFAULT_TIMEOUT_MS: u64 = 30_000;
9pub const MAX_TIMEOUT_MS: u64 = 60_000;
10pub const MAX_CONTENT_LENGTH: usize = 50_000;
11const USER_AGENT: &str = "Mozilla/5.0 (compatible; MCP-Lexicon/1.0)";
12
13/// Input parameters for the `web_fetch` tool
14#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
15#[serde(rename_all = "camelCase")]
16pub struct WebFetchInput {
17    /// The URL to fetch content from (must be a valid HTTP/HTTPS URL)
18    pub url: String,
19
20    /// A prompt describing what information to extract from the page (optional).
21    /// This is for documentation purposes and helps focus the user's intent.
22    pub prompt: Option<String>,
23
24    /// Optional timeout in milliseconds (default: 30000, max: 60000)
25    pub timeout: Option<u64>,
26}
27
28/// Output from the `web_fetch` tool
29#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
30#[serde(rename_all = "camelCase")]
31pub struct WebFetchOutput {
32    /// The fetched content converted to markdown
33    pub content: String,
34
35    /// The final URL after any redirects
36    pub final_url: String,
37
38    /// HTTP status code
39    pub status_code: u16,
40
41    /// Whether the content was truncated
42    pub truncated: bool,
43
44    /// Page title if available
45    pub title: Option<String>,
46
47    /// Author or byline if available
48    #[serde(skip_serializing_if = "Option::is_none")]
49    pub byline: Option<String>,
50
51    /// Display metadata for human-friendly rendering
52    #[serde(rename = "_meta", skip_serializing_if = "Option::is_none")]
53    #[schemars(skip)]
54    pub meta: Option<mcp_utils::display_meta::ToolResultMeta>,
55}
56
57/// Response from an HTTP client fetch operation
58#[derive(Debug, Clone)]
59pub struct HttpResponse {
60    pub final_url: String,
61    pub status_code: u16,
62    pub body: String,
63}
64
65/// Trait for HTTP clients that can fetch web content
66pub trait HttpClient: Send + Sync {
67    fn fetch(&self, url: &str, timeout: Duration) -> impl Future<Output = Result<HttpResponse, WebFetchError>> + Send;
68}
69
70/// Production HTTP client using reqwest
71#[derive(Debug, Clone)]
72pub struct ReqwestClient {
73    client: reqwest::Client,
74}
75
76impl Default for ReqwestClient {
77    fn default() -> Self {
78        Self::new()
79    }
80}
81
82impl ReqwestClient {
83    pub fn new() -> Self {
84        let client = reqwest::Client::builder().user_agent(USER_AGENT).build().expect("Failed to build HTTP client");
85
86        Self { client }
87    }
88}
89
90impl HttpClient for ReqwestClient {
91    async fn fetch(&self, url: &str, timeout: Duration) -> Result<HttpResponse, WebFetchError> {
92        let response = self.client.get(url).timeout(timeout).send().await.map_err(|e| {
93            if e.is_timeout() {
94                WebFetchError::Timeout(u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX))
95            } else {
96                WebFetchError::RequestFailed(e.to_string())
97            }
98        })?;
99
100        let final_url = response.url().to_string();
101        let status_code = response.status().as_u16();
102
103        let body = response.text().await.map_err(|e| WebFetchError::RequestFailed(e.to_string()))?;
104
105        Ok(HttpResponse { final_url, status_code, body })
106    }
107}
108
109/// Fake HTTP client for testing without network calls
110#[cfg(test)]
111#[derive(Debug, Clone)]
112pub struct FakeHttpClient {
113    responses: std::sync::Arc<std::sync::Mutex<std::collections::HashMap<String, HttpResponse>>>,
114    fetch_history: std::sync::Arc<std::sync::Mutex<Vec<String>>>,
115    default_response: Option<HttpResponse>,
116}
117
118#[cfg(test)]
119impl Default for FakeHttpClient {
120    fn default() -> Self {
121        Self::new()
122    }
123}
124
125#[cfg(test)]
126impl FakeHttpClient {
127    pub fn new() -> Self {
128        Self {
129            responses: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
130            fetch_history: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())),
131            default_response: None,
132        }
133    }
134
135    pub fn with_response(self, url: &str, response: HttpResponse) -> Self {
136        self.responses.lock().unwrap().insert(url.to_string(), response);
137        self
138    }
139
140    pub fn with_html(self, url: &str, html: &str) -> Self {
141        self.with_response(url, HttpResponse { final_url: url.to_string(), status_code: 200, body: html.to_string() })
142    }
143
144    pub fn with_default(mut self, response: HttpResponse) -> Self {
145        self.default_response = Some(response);
146        self
147    }
148
149    pub fn fetch_count(&self) -> usize {
150        self.fetch_history.lock().unwrap().len()
151    }
152
153    pub fn fetched_urls(&self) -> Vec<String> {
154        self.fetch_history.lock().unwrap().clone()
155    }
156}
157
158#[cfg(test)]
159impl HttpClient for FakeHttpClient {
160    async fn fetch(&self, url: &str, _timeout: Duration) -> Result<HttpResponse, WebFetchError> {
161        self.fetch_history.lock().unwrap().push(url.to_string());
162
163        let responses = self.responses.lock().unwrap();
164        if let Some(response) = responses.get(url) {
165            Ok(response.clone())
166        } else if let Some(ref default) = self.default_response {
167            Ok(default.clone())
168        } else {
169            Err(WebFetchError::RequestFailed(format!("No fake response configured for URL: {url}")))
170        }
171    }
172}
mcp_servers/coding/tools/web_fetch/http_client.rs

mcp_servers/coding/tools/web_fetch/
http_client.rs
