recursive/tools/
web_fetch.rs

1//! `web_fetch`: HTTP GET tool for fetching web content.
2//!
3//! Supports plain text and HTML content with optional truncation.
4
5use async_trait::async_trait;
6use reqwest::Client;
7use serde_json::{json, Value};
8use std::time::Duration;
9
10use super::Tool;
11use crate::error::{Error, Result};
12use crate::llm::ToolSpec;
13
14const DEFAULT_MAX_BYTES: usize = 65536;
15const REQUEST_TIMEOUT_SECS: u64 = 15;
16const CONNECT_TIMEOUT_SECS: u64 = 5;
17
18#[derive(Debug, Clone)]
19pub struct WebFetch {
20    client: Client,
21}
22
23impl WebFetch {
24    #[allow(clippy::new_without_default)]
25    pub fn new() -> Self {
26        let client = Client::builder()
27            .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
28            .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
29            .user_agent(format!("recursive-agent/{}", env!("CARGO_PKG_VERSION")))
30            .build()
31            .expect("reqwest client build");
32        Self { client }
33    }
34
35    /// Validate URL starts with http:// or https://
36    fn validate_url(url: &str) -> Result<String> {
37        if !url.starts_with("http://") && !url.starts_with("https://") {
38            return Err(Error::BadToolArgs {
39                name: "web_fetch".into(),
40                message: "URL must start with http:// or https://".into(),
41            });
42        }
43        Ok(url.to_string())
44    }
45
46    /// Simple HTML to markdown conversion - handles links, headings, basic tags.
47    fn html_to_markdown(html: &str) -> String {
48        let mut result = String::new();
49        let mut in_tag = false;
50        let mut chars = html.chars().peekable();
51        let mut link_href: Option<String> = None;
52        let mut link_text = String::new();
53        let mut in_link = false;
54
55        while let Some(c) = chars.next() {
56            if c == '<' {
57                // Check for closing </a> tag before processing new tag
58                if in_link {
59                    // End of link - output as markdown
60                    if let Some(href) = link_href.take() {
61                        let txt = link_text.trim();
62                        if !txt.is_empty() {
63                            result.push_str(&format!("[{}]({})", txt, href));
64                        }
65                    }
66                    link_text.clear();
67                    in_link = false;
68                }
69                in_tag = true;
70                // Look ahead to see what tag this is
71                let mut tag_buf = String::new();
72                let mut tag_chars = chars.clone();
73                while let Some(&nc) = tag_chars.peek() {
74                    if nc == '>' {
75                        break;
76                    }
77                    tag_buf.push(nc);
78                    tag_chars.next();
79                }
80                let tag_lower = tag_buf.to_lowercase();
81
82                // Handle opening <a> tag
83                if tag_lower.starts_with("a ") || tag_lower == "a>" {
84                    in_link = true;
85                    // Extract href if present - search within tag_buf for href="..."
86                    if let Some(start) = tag_buf.find("href=\"") {
87                        let url_start = start + 6; // skip href="
88                                                   // Find the closing quote
89                        if let Some(end_quote) = tag_buf[url_start..].find('"') {
90                            link_href = Some(tag_buf[url_start..url_start + end_quote].to_string());
91                        }
92                    }
93                }
94
95                // Handle heading opening tags - output marker immediately
96                if tag_lower.starts_with("h1") {
97                    result.push_str("# ");
98                } else if tag_lower.starts_with("h2") {
99                    result.push_str("## ");
100                } else if tag_lower.starts_with("h3") {
101                    result.push_str("### ");
102                }
103
104                // Skip content in script/style tags
105                if tag_lower.starts_with("script") || tag_lower.starts_with("style") {
106                    // Skip until </script> or </style>
107                    let close_tag = if tag_lower.starts_with("script") {
108                        "</script>"
109                    } else {
110                        "</style>"
111                    };
112                    let remaining: String = chars.clone().collect();
113                    if let Some(pos) = remaining.to_lowercase().find(close_tag) {
114                        for _ in 0..pos + close_tag.len() {
115                            chars.next();
116                        }
117                    }
118                    in_tag = false;
119                }
120                continue;
121            }
122
123            if in_tag {
124                if c == '>' {
125                    in_tag = false;
126                    // Check for block-level closing tags that add newlines
127                    let remaining: String = chars.clone().take(10).collect();
128                    let remaining_lower = remaining.to_lowercase();
129                    if remaining_lower.starts_with("</p>")
130                        || remaining_lower.starts_with("</div>")
131                        || remaining_lower.starts_with("</li>")
132                        || remaining_lower.starts_with("</h1>")
133                        || remaining_lower.starts_with("</h2>")
134                        || remaining_lower.starts_with("</h3>")
135                    {
136                        result.push('\n');
137                    }
138                    if remaining_lower.starts_with("<br") {
139                        result.push('\n');
140                    }
141                }
142                // Don't collect text while inside the tag - just skip it
143                continue;
144            }
145
146            // Regular text - when in a link, collect for link; otherwise output
147            if in_link {
148                link_text.push(c);
149            } else {
150                if c.is_whitespace() {
151                    if !result.ends_with(' ') && !result.ends_with('\n') {
152                        result.push(' ');
153                    }
154                } else {
155                    result.push(c);
156                }
157            }
158        }
159
160        // Handle any trailing link
161        if in_link {
162            if let Some(href) = link_href.take() {
163                let txt = link_text.trim();
164                if !txt.is_empty() {
165                    result.push_str(&format!("[{}]({})", txt, href));
166                }
167            }
168        }
169
170        // Clean up: collapse multiple spaces, remove leading/trailing whitespace per line
171        let lines: Vec<String> = result
172            .lines()
173            .map(|line| {
174                let trimmed = line.trim();
175                if trimmed.is_empty() {
176                    String::new()
177                } else {
178                    // Collapse multiple spaces
179                    let mut collapsed = String::new();
180                    let mut last_was_space = false;
181                    for c in trimmed.chars() {
182                        if c.is_whitespace() {
183                            if !last_was_space {
184                                collapsed.push(' ');
185                                last_was_space = true;
186                            }
187                        } else {
188                            collapsed.push(c);
189                            last_was_space = false;
190                        }
191                    }
192                    collapsed
193                }
194            })
195            .collect();
196
197        lines.join("\n")
198    }
199}
200
201#[async_trait]
202impl Tool for WebFetch {
203    fn spec(&self) -> ToolSpec {
204        ToolSpec {
205            name: "web_fetch".into(),
206            description: "Fetch content from a URL via HTTP GET. Returns the body as text, optionally truncated to max_bytes. For HTML pages, attempts basic markdown conversion."
207                .into(),
208            parameters: json!({
209                "type": "object",
210                "properties": {
211                    "url": { "type": "string", "description": "URL to fetch. Must start with http:// or https://." },
212                    "max_bytes": { "type": "integer", "description": "Maximum bytes to read from body. Defaults to 65536. Truncation adds a note." }
213                },
214                "required": ["url"]
215            }),
216        }
217    }
218
219    fn is_readonly(&self) -> bool {
220        true
221    }
222
223    async fn execute(&self, args: Value) -> Result<String> {
224        let url = args["url"].as_str().ok_or_else(|| Error::BadToolArgs {
225            name: "web_fetch".into(),
226            message: "missing `url`".into(),
227        })?;
228
229        let validated_url = Self::validate_url(url)?;
230
231        let max_bytes = args
232            .get("max_bytes")
233            .and_then(|v| v.as_u64())
234            .map(|n| n as usize)
235            .unwrap_or(DEFAULT_MAX_BYTES);
236
237        let response = self
238            .client
239            .get(&validated_url)
240            .send()
241            .await
242            .map_err(|e| Error::Tool {
243                name: "web_fetch".into(),
244                message: format!("request failed: {}", e),
245            })?;
246
247        let status = response.status();
248        if !status.is_success() {
249            let body = response.text().await.unwrap_or_default();
250            let excerpt = if body.len() > 200 {
251                format!("{}...", &body[..200])
252            } else {
253                body
254            };
255            return Err(Error::Tool {
256                name: "web_fetch".into(),
257                message: format!("HTTP {}: {}", status.as_u16(), excerpt),
258            });
259        }
260
261        let content_type = response
262            .headers()
263            .get("content-type")
264            .and_then(|v| v.to_str().ok())
265            .unwrap_or("")
266            .to_string();
267
268        let body = response.text().await.map_err(|e| Error::Tool {
269            name: "web_fetch".into(),
270            message: format!("failed to read response body: {}", e),
271        })?;
272
273        let total_bytes = body.len();
274        let truncated = if total_bytes > max_bytes {
275            let truncated_body = &body[..max_bytes];
276            let msg = format!(
277                "{}\n\n[…truncated at {} bytes; total body was {} bytes]",
278                truncated_body, max_bytes, total_bytes
279            );
280            msg
281        } else {
282            body
283        };
284
285        // Convert HTML to markdown if content type suggests HTML
286        if content_type.contains("text/html") {
287            return Ok(Self::html_to_markdown(&truncated));
288        }
289
290        Ok(truncated)
291    }
292}
293
294#[cfg(test)]
295mod tests {
296    use super::*;
297
298    #[test]
299    fn validate_url_rejects_invalid() {
300        assert!(WebFetch::validate_url("ftp://example.com").is_err());
301        assert!(WebFetch::validate_url("example.com").is_err());
302        assert!(WebFetch::validate_url("/path").is_err());
303    }
304
305    #[test]
306    fn validate_url_accepts_valid() {
307        assert!(WebFetch::validate_url("http://example.com").is_ok());
308        assert!(WebFetch::validate_url("https://example.com").is_ok());
309    }
310
311    #[test]
312    fn html_to_markdown_strips_scripts() {
313        let html = "<html><script>alert('x')</script><body>Hello</body></html>";
314        let md = WebFetch::html_to_markdown(html);
315        assert!(!md.contains("alert"));
316        assert!(md.contains("Hello"));
317    }
318
319    #[test]
320    fn html_to_markdown_preserves_links() {
321        let html = "<a href=\"https://example.com\">Example</a>";
322        let md = WebFetch::html_to_markdown(html);
323        assert!(md.contains("[Example](https://example.com)"));
324    }
325
326    #[test]
327    fn html_to_markdown_preserves_headings() {
328        let html = "<h1>Title</h1><p>Para</p>";
329        let md = WebFetch::html_to_markdown(html);
330        assert!(md.contains("# Title"));
331        assert!(md.contains("Para"));
332    }
333
334    #[test]
335    fn html_to_markdown_collapse_whitespace() {
336        let html = "<p>Hello    World</p>";
337        let md = WebFetch::html_to_markdown(html);
338        assert!(md.contains("Hello World"));
339    }
340
341    #[test]
342    fn collapse_whitespace_basic() {
343        // Test the internal behavior by checking result
344        let html = "Hello   World";
345        let md = WebFetch::html_to_markdown(html);
346        assert!(md.contains("Hello World"));
347    }
348
349    #[tokio::test]
350    async fn test_a_mock_server_returns_text_plain() {
351        // Spawn mock server
352        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
353        let addr = listener.local_addr().unwrap();
354
355        let handle = std::thread::spawn(move || {
356            let (mut stream, _) = listener.accept().unwrap();
357            use std::io::{Read, Write};
358            let mut buf = [0u8; 4096];
359            let _ = stream.read(&mut buf);
360
361            let body = "Hello, world!";
362            let response = format!(
363                "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
364                body.len(),
365                body
366            );
367            write!(stream, "{}", response).unwrap();
368            stream.flush().unwrap();
369        });
370
371        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
372
373        let client = Client::builder()
374            .timeout(Duration::from_secs(2))
375            .connect_timeout(Duration::from_secs(1))
376            .build()
377            .unwrap();
378
379        let result = client.get(format!("http://{addr}")).send().await;
380
381        handle.join().ok();
382
383        assert!(result.is_ok());
384        let resp = result.unwrap();
385        assert!(resp.status().is_success());
386        let body = resp.text().await.unwrap();
387        assert_eq!(body, "Hello, world!");
388    }
389
390    #[tokio::test]
391    async fn test_b_mock_server_returns_404() {
392        // Spawn mock server returning 404
393        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
394        let addr = listener.local_addr().unwrap();
395
396        let handle = std::thread::spawn(move || {
397            let (mut stream, _) = listener.accept().unwrap();
398            use std::io::{Read, Write};
399            let mut buf = [0u8; 4096];
400            let _ = stream.read(&mut buf);
401
402            let body = "Not Found";
403            let response = format!(
404                "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
405                body.len(),
406                body
407            );
408            write!(stream, "{}", response).unwrap();
409            stream.flush().unwrap();
410        });
411
412        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
413
414        let client = Client::builder()
415            .timeout(Duration::from_secs(2))
416            .connect_timeout(Duration::from_secs(1))
417            .build()
418            .unwrap();
419
420        let result = client.get(format!("http://{addr}")).send().await;
421
422        handle.join().ok();
423
424        assert!(result.is_ok());
425        let resp = result.unwrap();
426        assert_eq!(resp.status().as_u16(), 404);
427    }
428
429    #[tokio::test]
430    async fn test_c_body_exceeds_max_bytes() {
431        // Spawn mock server returning large body
432        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
433        let addr = listener.local_addr().unwrap();
434
435        let body = "a".repeat(200);
436        let handle = std::thread::spawn(move || {
437            let (mut stream, _) = listener.accept().unwrap();
438            use std::io::{Read, Write};
439            let mut buf = [0u8; 4096];
440            let _ = stream.read(&mut buf);
441
442            let response = format!(
443                "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
444                body.len(),
445                body
446            );
447            write!(stream, "{}", response).unwrap();
448            stream.flush().unwrap();
449        });
450
451        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
452
453        let tool = WebFetch::new();
454        let result = tool
455            .execute(json!({
456                "url": format!("http://{}/test", addr),
457                "max_bytes": 50
458            }))
459            .await;
460
461        handle.join().ok();
462
463        let output = result.expect("should succeed");
464        // Should contain truncated body
465        assert!(output.contains("aaaaaaaa"));
466        // Should contain truncation marker
467        assert!(output.contains("truncated"));
468        // Should mention original size
469        assert!(output.contains("200 bytes"));
470    }
471
472    #[tokio::test]
473    async fn web_fetch_tool_on_mock_server() {
474        // Test the full tool with mock server
475        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
476        let addr = listener.local_addr().unwrap();
477
478        let body = "Plain text content";
479        let handle = std::thread::spawn(move || {
480            let (mut stream, _) = listener.accept().unwrap();
481            use std::io::{Read, Write};
482            let mut buf = [0u8; 4096];
483            let _ = stream.read(&mut buf);
484
485            let response = format!(
486                "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
487                body.len(),
488                body
489            );
490            write!(stream, "{}", response).unwrap();
491            stream.flush().unwrap();
492        });
493
494        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
495
496        let tool = WebFetch::new();
497        let result = tool
498            .execute(json!({ "url": format!("http://{}/test", addr) }))
499            .await;
500
501        handle.join().ok();
502
503        let output = result.expect("should succeed");
504        assert!(output.contains("Plain text content"));
505    }
506
507    #[tokio::test]
508    async fn web_fetch_rejects_invalid_url() {
509        let tool = WebFetch::new();
510        let result = tool
511            .execute(json!({ "url": "not-a-url" }))
512            .await
513            .unwrap_err();
514        assert!(result.to_string().contains("http:// or https://"));
515    }
516
517    #[tokio::test]
518    async fn web_fetch_rejects_missing_url() {
519        let tool = WebFetch::new();
520        let result = tool.execute(json!({})).await.unwrap_err();
521        assert!(result.to_string().contains("missing `url`"));
522    }
523
524    #[tokio::test]
525    async fn web_fetch_handles_html_content_type() {
526        // Test HTML to markdown conversion
527        let html = "<html><body><h1>Title</h1><p>Hello <a href=\"http://ex.com\">Link</a></p></body></html>";
528        let md = WebFetch::html_to_markdown(html);
529        assert!(md.contains("# Title"));
530        assert!(md.contains("Link"));
531    }
532}
recursive/tools/web_fetch.rs

recursive/tools/
web_fetch.rs
