opendev_tools_impl/web_fetch/
mod.rs

1//! Web fetch tool — fetch URL content via HTTP.
2//!
3//! Supports optional HTML-to-markdown extraction for LLM-friendly output,
4//! mirroring Python's `extract_text` parameter.
5
6mod html_converter;
7
8use std::collections::HashMap;
9
10use opendev_tools_core::{BaseTool, ToolContext, ToolResult};
11
12use html_converter::html_to_markdown;
13
14/// Maximum response body size (1 MB).
15const MAX_BODY_SIZE: usize = 1_024 * 1_024;
16
17/// Maximum timeout (120 seconds).
18const MAX_TIMEOUT_SECS: u64 = 120;
19
20/// Default timeout (30 seconds).
21const DEFAULT_TIMEOUT_SECS: u64 = 30;
22
23/// Tool for fetching web page content.
24#[derive(Debug)]
25pub struct WebFetchTool;
26
27#[async_trait::async_trait]
28impl BaseTool for WebFetchTool {
29    fn name(&self) -> &str {
30        "web_fetch"
31    }
32
33    fn description(&self) -> &str {
34        "Fetch the content of a URL. Supports optional HTML-to-markdown extraction for clean, LLM-friendly output."
35    }
36
37    fn parameter_schema(&self) -> serde_json::Value {
38        serde_json::json!({
39            "type": "object",
40            "properties": {
41                "url": {
42                    "type": "string",
43                    "description": "URL to fetch"
44                },
45                "headers": {
46                    "type": "object",
47                    "description": "Optional HTTP headers as key-value pairs"
48                },
49                "extract_markdown": {
50                    "type": "boolean",
51                    "description": "Convert HTML to clean markdown for easier reading (default: true for HTML content)"
52                },
53                "format": {
54                    "type": "string",
55                    "enum": ["text", "markdown", "html"],
56                    "description": "Output format: 'text' for plain text, 'markdown' for HTML-to-markdown conversion (default for HTML), 'html' for raw HTML"
57                },
58                "timeout": {
59                    "type": "number",
60                    "description": "Request timeout in seconds (default: 30, max: 120)"
61                }
62            },
63            "required": ["url"]
64        })
65    }
66
67    async fn execute(
68        &self,
69        args: HashMap<String, serde_json::Value>,
70        _ctx: &ToolContext,
71    ) -> ToolResult {
72        let url = match args.get("url").and_then(|v| v.as_str()) {
73            Some(u) => u,
74            None => return ToolResult::fail("url is required"),
75        };
76
77        // Basic URL validation
78        if !url.starts_with("http://") && !url.starts_with("https://") {
79            return ToolResult::fail("URL must start with http:// or https://");
80        }
81
82        // Parse timeout (capped at MAX_TIMEOUT_SECS).
83        let timeout_secs = args
84            .get("timeout")
85            .and_then(|v| v.as_u64())
86            .map(|t| t.min(MAX_TIMEOUT_SECS))
87            .unwrap_or(DEFAULT_TIMEOUT_SECS);
88
89        // Parse format parameter.
90        let format = args
91            .get("format")
92            .and_then(|v| v.as_str())
93            .unwrap_or("markdown");
94
95        let client = reqwest::Client::builder()
96            .connect_timeout(std::time::Duration::from_secs(10))
97            .timeout(std::time::Duration::from_secs(timeout_secs))
98            .redirect(reqwest::redirect::Policy::limited(5))
99            .build();
100
101        let client = match client {
102            Ok(c) => c,
103            Err(e) => return ToolResult::fail(format!("Failed to create HTTP client: {e}")),
104        };
105
106        // Build Accept header based on format.
107        let accept_header = match format {
108            "html" => "text/html,application/xhtml+xml,*/*;q=0.8",
109            "text" => "text/plain,text/html;q=0.5,*/*;q=0.3",
110            _ => "text/html,application/xhtml+xml,text/plain;q=0.8,*/*;q=0.5", // markdown (default)
111        };
112
113        let mut request = client
114            .get(url)
115            .header("Accept", accept_header)
116            .header("Accept-Language", "en-US,en;q=0.9");
117
118        // Add custom headers (may override Accept/Accept-Language).
119        if let Some(headers) = args.get("headers").and_then(|v| v.as_object()) {
120            for (key, value) in headers {
121                if let Some(val) = value.as_str() {
122                    request = request.header(key.as_str(), val);
123                }
124            }
125        }
126
127        let response = match request.send().await {
128            Ok(r) => r,
129            Err(e) => return ToolResult::fail(format!("Request failed: {e}")),
130        };
131
132        let status = response.status().as_u16();
133
134        // Detect Cloudflare bot challenge: 403 with cf-mitigated header.
135        let is_cf_blocked = status == 403
136            && response
137                .headers()
138                .get("cf-mitigated")
139                .and_then(|v| v.to_str().ok())
140                .is_some_and(|v| v.contains("challenge"));
141
142        let content_type = response
143            .headers()
144            .get("content-type")
145            .and_then(|v| v.to_str().ok())
146            .unwrap_or("unknown")
147            .to_string();
148
149        let body = match response.text().await {
150            Ok(t) => t,
151            Err(e) => return ToolResult::fail(format!("Failed to read response body: {e}")),
152        };
153
154        // Retry with simpler User-Agent if Cloudflare blocked us.
155        let (status, content_type, body) = if is_cf_blocked {
156            tracing::debug!("Cloudflare challenge detected, retrying with simpler UA");
157            let retry = client
158                .get(url)
159                .header("User-Agent", "opendev")
160                .header("Accept", accept_header)
161                .header("Accept-Language", "en-US,en;q=0.9")
162                .send()
163                .await;
164            match retry {
165                Ok(r) => {
166                    let s = r.status().as_u16();
167                    let ct = r
168                        .headers()
169                        .get("content-type")
170                        .and_then(|v| v.to_str().ok())
171                        .unwrap_or("unknown")
172                        .to_string();
173                    let b = r.text().await.unwrap_or_default();
174                    (s, ct, b)
175                }
176                Err(_) => (status, content_type, body), // fall back to original
177            }
178        } else {
179            (status, content_type, body)
180        };
181
182        // Determine if we should extract markdown based on format and extract_markdown params.
183        let extract_markdown = match format {
184            "html" => false, // raw HTML requested
185            "text" => false, // plain text, no conversion
186            _ => {
187                // "markdown" or default: respect extract_markdown param or auto-detect
188                args.get("extract_markdown")
189                    .and_then(|v| v.as_bool())
190                    .unwrap_or(content_type.contains("html"))
191            }
192        };
193
194        // Convert HTML to markdown if requested and content is HTML
195        let body = if extract_markdown && content_type.contains("html") {
196            html_to_markdown(&body)
197        } else {
198            body
199        };
200
201        let truncated = body.len() > MAX_BODY_SIZE;
202        let body = if truncated {
203            format!(
204                "{}...\n\n[truncated, showing first {} bytes of {}]",
205                &body[..MAX_BODY_SIZE],
206                MAX_BODY_SIZE,
207                body.len()
208            )
209        } else {
210            body
211        };
212
213        let mut metadata = HashMap::new();
214        metadata.insert("status".into(), serde_json::json!(status));
215        metadata.insert("content_type".into(), serde_json::json!(content_type));
216        metadata.insert("truncated".into(), serde_json::json!(truncated));
217        metadata.insert(
218            "extracted_markdown".into(),
219            serde_json::json!(extract_markdown),
220        );
221
222        if status >= 400 {
223            return ToolResult {
224                success: false,
225                output: Some(body),
226                error: Some(format!("HTTP {status}")),
227                metadata,
228                duration_ms: None,
229                llm_suffix: None,
230            };
231        }
232
233        ToolResult::ok_with_metadata(body, metadata)
234    }
235}
236
237#[cfg(test)]
238mod tests;
opendev_tools_impl/web_fetch/mod.rs

opendev_tools_impl/web_fetch/
mod.rs
