mcp_servers/coding/tools/web_fetch/
http_client.rs

1use futures::future::BoxFuture;
2use reqwest::header::CONTENT_TYPE;
3use schemars::JsonSchema;
4use serde::{Deserialize, Serialize};
5use std::time::Duration;
6
7use crate::coding::error::WebFetchError;
8
9pub const DEFAULT_TIMEOUT_MS: u64 = 30_000;
10pub const MAX_TIMEOUT_MS: u64 = 60_000;
11const USER_AGENT: &str = "Mozilla/5.0 (compatible; MCP-Lexicon/1.0)";
12
13/// Input parameters for the `web_fetch` tool
14#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
15#[serde(rename_all = "camelCase")]
16pub struct WebFetchInput {
17    /// The URL to fetch content from (must be a valid HTTP/HTTPS URL)
18    pub url: String,
19
20    /// A prompt describing what information to extract from the page (optional).
21    /// This is for documentation purposes and helps focus the user's intent.
22    pub prompt: Option<String>,
23
24    /// Optional timeout in milliseconds (default: 30000, max: 60000)
25    pub timeout: Option<u64>,
26}
27
28/// Output from the `web_fetch` tool
29#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
30#[serde(rename_all = "camelCase")]
31pub struct WebFetchOutput {
32    /// The fetched content converted to markdown
33    pub content: String,
34
35    /// The final URL after any redirects
36    pub final_url: String,
37
38    /// HTTP status code
39    pub status_code: u16,
40
41    /// Page title if available
42    pub title: Option<String>,
43
44    /// Author or byline if available
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub byline: Option<String>,
47}
48
49/// Response from an HTTP client fetch operation
50#[derive(Debug, Clone)]
51pub struct HttpResponse {
52    pub final_url: String,
53    pub status_code: u16,
54    pub body: String,
55    pub content_type: Option<String>,
56}
57
58/// Trait for HTTP clients that can fetch web content
59pub trait HttpClient: Send + Sync {
60    fn fetch<'a>(&'a self, url: &'a str, timeout: Duration) -> BoxFuture<'a, Result<HttpResponse, WebFetchError>>;
61}
62
63/// Production HTTP client using reqwest
64#[derive(Debug, Clone)]
65pub struct ReqwestClient {
66    client: reqwest::Client,
67}
68
69impl Default for ReqwestClient {
70    fn default() -> Self {
71        Self::new()
72    }
73}
74
75impl ReqwestClient {
76    pub fn new() -> Self {
77        let client = reqwest::Client::builder().user_agent(USER_AGENT).build().expect("Failed to build HTTP client");
78
79        Self { client }
80    }
81}
82
83impl HttpClient for ReqwestClient {
84    fn fetch<'a>(&'a self, url: &'a str, timeout: Duration) -> BoxFuture<'a, Result<HttpResponse, WebFetchError>> {
85        Box::pin(async move {
86            let response = self.client.get(url).timeout(timeout).send().await.map_err(|e| {
87                if e.is_timeout() {
88                    WebFetchError::Timeout(u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX))
89                } else {
90                    WebFetchError::RequestFailed(e.to_string())
91                }
92            })?;
93
94            let final_url = response.url().to_string();
95            let status_code = response.status().as_u16();
96            let content_type =
97                response.headers().get(CONTENT_TYPE).and_then(|value| value.to_str().ok()).map(str::to_owned);
98
99            let body = response.text().await.map_err(|e| WebFetchError::RequestFailed(e.to_string()))?;
100
101            Ok(HttpResponse { final_url, status_code, body, content_type })
102        })
103    }
104}
mcp_servers/coding/tools/web_fetch/http_client.rs

mcp_servers/coding/tools/web_fetch/
http_client.rs
