1use async_trait::async_trait;
6use reqwest::Client;
7use serde_json::{json, Value};
8use std::time::Duration;
9
10use super::Tool;
11use crate::acp::ToolKind;
12use crate::error::{Error, Result};
13use crate::llm::ToolSpec;
14use crate::tools::url_guard::validate_url;
15
16const DEFAULT_MAX_BYTES: usize = 65536;
17const REQUEST_TIMEOUT_SECS: u64 = 15;
18const CONNECT_TIMEOUT_SECS: u64 = 5;
19
20#[derive(Debug, Clone)]
21pub struct WebFetch {
22 client: Client,
23}
24
25impl Default for WebFetch {
26 fn default() -> Self {
27 Self::new()
28 }
29}
30
31impl WebFetch {
32 pub fn new() -> Self {
33 let client = Client::builder()
34 .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
35 .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
36 .user_agent(format!("recursive-agent/{}", env!("CARGO_PKG_VERSION")))
37 .redirect(reqwest::redirect::Policy::none())
44 .build();
49 #[allow(
50 clippy::expect_used,
51 reason = "TLS backend unavailable is a fatal startup error"
52 )]
53 let client = client.expect("reqwest client build: TLS backend unavailable");
54 Self { client }
55 }
56
57 async fn fetch(&self, validated_url: &str, max_bytes: usize) -> Result<String> {
63 let response = self
64 .client
65 .get(validated_url)
66 .send()
67 .await
68 .map_err(|e| Error::Tool {
69 name: "WebFetch".into(),
70 call_id: None,
71 message: format!("request failed: {}", e),
72 })?;
73
74 let status = response.status();
75 if status.is_redirection() {
79 let location = response
80 .headers()
81 .get("location")
82 .and_then(|v| v.to_str().ok())
83 .unwrap_or("<none>");
84 return Err(Error::Tool {
85 name: "WebFetch".into(),
86 call_id: None,
87 message: format!(
88 "WebFetch does not follow redirects; got HTTP {} (Location: {})",
89 status.as_u16(),
90 location
91 ),
92 });
93 }
94 if !status.is_success() {
95 let body = response.text().await.unwrap_or_default();
96 let excerpt = if body.len() > 200 {
97 format!("{}...", crate::truncate_str(&body, 200))
98 } else {
99 body
100 };
101 return Err(Error::Tool {
102 name: "WebFetch".into(),
103 call_id: None,
104 message: format!("HTTP {}: {}", status.as_u16(), excerpt),
105 });
106 }
107
108 let content_type = response
109 .headers()
110 .get("content-type")
111 .and_then(|v| v.to_str().ok())
112 .unwrap_or("")
113 .to_string();
114
115 let body = response.text().await.map_err(|e| Error::Tool {
116 name: "WebFetch".into(),
117 call_id: None,
118 message: format!("failed to read response body: {}", e),
119 })?;
120
121 let total_bytes = body.len();
122 let truncated = if total_bytes > max_bytes {
123 let truncated_body = crate::truncate_str(&body, max_bytes);
124 format!(
125 "{}\n\n[…truncated at {} bytes; total body was {} bytes]",
126 truncated_body, max_bytes, total_bytes
127 )
128 } else {
129 body
130 };
131
132 if content_type.contains("text/html") {
134 return Ok(Self::html_to_markdown(&truncated));
135 }
136
137 Ok(truncated)
138 }
139
140 fn html_to_markdown(html: &str) -> String {
142 let mut result = String::new();
143 let mut in_tag = false;
144 let mut chars = html.chars().peekable();
145 let mut link_href: Option<String> = None;
146 let mut link_text = String::new();
147 let mut in_link = false;
148
149 while let Some(c) = chars.next() {
150 if c == '<' {
151 if in_link {
153 if let Some(href) = link_href.take() {
155 let txt = link_text.trim();
156 if !txt.is_empty() {
157 result.push_str(&format!("[{}]({})", txt, href));
158 }
159 }
160 link_text.clear();
161 in_link = false;
162 }
163 in_tag = true;
164 let mut tag_buf = String::new();
166 let mut tag_chars = chars.clone();
167 while let Some(&nc) = tag_chars.peek() {
168 if nc == '>' {
169 break;
170 }
171 tag_buf.push(nc);
172 tag_chars.next();
173 }
174 let tag_lower = tag_buf.to_lowercase();
175
176 if tag_lower.starts_with("a ") || tag_lower == "a>" {
178 in_link = true;
179 if let Some(start) = tag_buf.find("href=\"") {
181 let url_start = start + 6; if let Some(end_quote) = tag_buf[url_start..].find('"') {
184 link_href = Some(tag_buf[url_start..url_start + end_quote].to_string());
185 }
186 }
187 }
188
189 if tag_lower.starts_with("h1") {
191 result.push_str("# ");
192 } else if tag_lower.starts_with("h2") {
193 result.push_str("## ");
194 } else if tag_lower.starts_with("h3") {
195 result.push_str("### ");
196 }
197
198 if tag_lower.starts_with("script") || tag_lower.starts_with("style") {
200 let close_tag = if tag_lower.starts_with("script") {
202 "</script>"
203 } else {
204 "</style>"
205 };
206 let remaining: String = chars.clone().collect();
207 if let Some(pos) = remaining.to_lowercase().find(close_tag) {
208 for _ in 0..pos + close_tag.len() {
209 chars.next();
210 }
211 }
212 in_tag = false;
213 }
214 continue;
215 }
216
217 if in_tag {
218 if c == '>' {
219 in_tag = false;
220 let remaining: String = chars.clone().take(10).collect();
222 let remaining_lower = remaining.to_lowercase();
223 if remaining_lower.starts_with("</p>")
224 || remaining_lower.starts_with("</div>")
225 || remaining_lower.starts_with("</li>")
226 || remaining_lower.starts_with("</h1>")
227 || remaining_lower.starts_with("</h2>")
228 || remaining_lower.starts_with("</h3>")
229 {
230 result.push('\n');
231 }
232 if remaining_lower.starts_with("<br") {
233 result.push('\n');
234 }
235 }
236 continue;
238 }
239
240 if in_link {
242 link_text.push(c);
243 } else {
244 if c.is_whitespace() {
245 if !result.ends_with(' ') && !result.ends_with('\n') {
246 result.push(' ');
247 }
248 } else {
249 result.push(c);
250 }
251 }
252 }
253
254 if in_link {
256 if let Some(href) = link_href.take() {
257 let txt = link_text.trim();
258 if !txt.is_empty() {
259 result.push_str(&format!("[{}]({})", txt, href));
260 }
261 }
262 }
263
264 let lines: Vec<String> = result
266 .lines()
267 .map(|line| {
268 let trimmed = line.trim();
269 if trimmed.is_empty() {
270 String::new()
271 } else {
272 let mut collapsed = String::new();
274 let mut last_was_space = false;
275 for c in trimmed.chars() {
276 if c.is_whitespace() {
277 if !last_was_space {
278 collapsed.push(' ');
279 last_was_space = true;
280 }
281 } else {
282 collapsed.push(c);
283 last_was_space = false;
284 }
285 }
286 collapsed
287 }
288 })
289 .collect();
290
291 lines.join("\n")
292 }
293}
294
295#[async_trait]
296impl Tool for WebFetch {
297 fn spec(&self) -> ToolSpec {
298 ToolSpec {
299 name: "WebFetch".into(),
300 description: "Fetch content from a URL via HTTP GET. Returns the body as text, optionally truncated to max_bytes. For HTML pages, attempts basic markdown conversion."
301 .into(),
302 parameters: json!({
303 "type": "object",
304 "properties": {
305 "url": { "type": "string", "description": "URL to fetch. Must start with http:// or https://." },
306 "max_bytes": { "type": "integer", "description": "Maximum bytes to read from body. Defaults to 65536. Truncation adds a note." }
307 },
308 "required": ["url"]
309 }),
310 }
311 }
312
313 fn is_deferred(&self) -> bool {
314 true
315 }
316
317 fn side_effect_class(&self) -> crate::tools::ToolSideEffect {
318 crate::tools::ToolSideEffect::ReadOnly
319 }
320
321 fn kind(&self) -> ToolKind {
322 ToolKind::Fetch
323 }
324
325 async fn execute(&self, args: Value) -> Result<String> {
326 let url = args["url"].as_str().ok_or_else(|| Error::BadToolArgs {
327 name: "WebFetch".into(),
328 message: "missing `url`".into(),
329 })?;
330
331 let validated_url = validate_url(url)?;
332
333 let max_bytes = args
334 .get("max_bytes")
335 .and_then(|v| v.as_u64())
336 .map(|n| n as usize)
337 .unwrap_or(DEFAULT_MAX_BYTES);
338
339 self.fetch(&validated_url, max_bytes).await
340 }
341}
342
343#[cfg(test)]
344mod tests {
345 use super::*;
346 use crate::tools::url_guard::is_private_ip;
347
348 #[test]
349 fn validate_url_rejects_invalid() {
350 assert!(validate_url("ftp://example.com").is_err());
351 assert!(validate_url("example.com").is_err());
352 assert!(validate_url("/path").is_err());
353 }
354
355 #[test]
356 fn validate_url_accepts_valid() {
357 assert!(validate_url("http://example.com").is_ok());
358 assert!(validate_url("https://example.com").is_ok());
359 }
360
361 #[test]
362 fn validate_url_blocks_ssrf_targets() {
363 assert!(validate_url("http://localhost/secret").is_err());
365 assert!(validate_url("http://localhost:8080/admin").is_err());
366 assert!(validate_url("http://127.0.0.1/").is_err());
368 assert!(validate_url("http://127.1.2.3/").is_err());
369 assert!(validate_url("http://169.254.169.254/latest/meta-data/").is_err());
371 assert!(validate_url("http://10.0.0.1/").is_err());
373 assert!(validate_url("http://192.168.1.1/").is_err());
374 assert!(validate_url("http://172.16.0.1/").is_err());
375 assert!(validate_url("http://[::1]/").is_err());
377 assert!(validate_url("http://metadata.google.internal/").is_err());
379 }
380
381 #[test]
382 fn html_to_markdown_strips_scripts() {
383 let html = "<html><script>alert('x')</script><body>Hello</body></html>";
384 let md = WebFetch::html_to_markdown(html);
385 assert!(!md.contains("alert"));
386 assert!(md.contains("Hello"));
387 }
388
389 #[test]
390 fn html_to_markdown_preserves_links() {
391 let html = "<a href=\"https://example.com\">Example</a>";
392 let md = WebFetch::html_to_markdown(html);
393 assert!(md.contains("[Example](https://example.com)"));
394 }
395
396 #[test]
397 fn html_to_markdown_preserves_headings() {
398 let html = "<h1>Title</h1><p>Para</p>";
399 let md = WebFetch::html_to_markdown(html);
400 assert!(md.contains("# Title"));
401 assert!(md.contains("Para"));
402 }
403
404 #[test]
405 fn html_to_markdown_collapse_whitespace() {
406 let html = "<p>Hello World</p>";
407 let md = WebFetch::html_to_markdown(html);
408 assert!(md.contains("Hello World"));
409 }
410
411 #[test]
412 fn collapse_whitespace_basic() {
413 let html = "Hello World";
415 let md = WebFetch::html_to_markdown(html);
416 assert!(md.contains("Hello World"));
417 }
418
419 #[test]
420 fn web_fetch_construction_smoke() {
421 let tool = WebFetch::new();
422 assert_eq!(tool.spec().name, "WebFetch");
423 }
424
425 #[tokio::test]
426 async fn test_a_mock_server_returns_text_plain() {
427 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
429 let addr = listener.local_addr().unwrap();
430
431 let handle = std::thread::spawn(move || {
432 let (mut stream, _) = listener.accept().unwrap();
433 use std::io::{Read, Write};
434 let mut buf = [0u8; 4096];
435 let _ = stream.read(&mut buf);
436
437 let body = "Hello, world!";
438 let response = format!(
439 "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
440 body.len(),
441 body
442 );
443 write!(stream, "{}", response).unwrap();
444 stream.flush().unwrap();
445 });
446
447 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
448
449 let client = Client::builder()
450 .timeout(Duration::from_secs(2))
451 .connect_timeout(Duration::from_secs(1))
452 .build()
453 .unwrap();
454
455 let result = client.get(format!("http://{addr}")).send().await;
456
457 handle.join().ok();
458
459 assert!(result.is_ok());
460 let resp = result.unwrap();
461 assert!(resp.status().is_success());
462 let body = resp.text().await.unwrap();
463 assert_eq!(body, "Hello, world!");
464 }
465
466 #[tokio::test]
467 async fn test_b_mock_server_returns_404() {
468 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
470 let addr = listener.local_addr().unwrap();
471
472 let handle = std::thread::spawn(move || {
473 let (mut stream, _) = listener.accept().unwrap();
474 use std::io::{Read, Write};
475 let mut buf = [0u8; 4096];
476 let _ = stream.read(&mut buf);
477
478 let body = "Not Found";
479 let response = format!(
480 "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
481 body.len(),
482 body
483 );
484 write!(stream, "{}", response).unwrap();
485 stream.flush().unwrap();
486 });
487
488 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
489
490 let client = Client::builder()
491 .timeout(Duration::from_secs(2))
492 .connect_timeout(Duration::from_secs(1))
493 .build()
494 .unwrap();
495
496 let result = client.get(format!("http://{addr}")).send().await;
497
498 handle.join().ok();
499
500 assert!(result.is_ok());
501 let resp = result.unwrap();
502 assert_eq!(resp.status().as_u16(), 404);
503 }
504
505 #[tokio::test]
506 async fn test_c_body_exceeds_max_bytes() {
507 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
509 let addr = listener.local_addr().unwrap();
510
511 let body = "a".repeat(200);
512 let handle = std::thread::spawn(move || {
513 let (mut stream, _) = listener.accept().unwrap();
514 use std::io::{Read, Write};
515 let mut buf = [0u8; 4096];
516 let _ = stream.read(&mut buf);
517
518 let response = format!(
519 "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
520 body.len(),
521 body
522 );
523 write!(stream, "{}", response).unwrap();
524 stream.flush().unwrap();
525 });
526
527 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
528
529 let tool = WebFetch::new();
530 let result = tokio::time::timeout(
534 Duration::from_secs(10),
535 tool.fetch(&format!("http://{addr}/test"), 50),
536 )
537 .await
538 .expect("fetch should not hang");
539
540 handle.join().ok();
541
542 let output = result.expect("should succeed");
543 assert!(output.contains("aaaaaaaa"));
545 assert!(output.contains("truncated"));
547 assert!(output.contains("200 bytes"));
549 }
550
551 #[tokio::test]
552 async fn web_fetch_tool_on_mock_server() {
553 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
555 let addr = listener.local_addr().unwrap();
556
557 let body = "Plain text content";
558 let handle = std::thread::spawn(move || {
559 let (mut stream, _) = listener.accept().unwrap();
560 use std::io::{Read, Write};
561 let mut buf = [0u8; 4096];
562 let _ = stream.read(&mut buf);
563
564 let response = format!(
565 "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
566 body.len(),
567 body
568 );
569 write!(stream, "{}", response).unwrap();
570 stream.flush().unwrap();
571 });
572
573 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
574
575 let tool = WebFetch::new();
576 let result = tokio::time::timeout(
580 Duration::from_secs(10),
581 tool.fetch(&format!("http://{addr}/test"), DEFAULT_MAX_BYTES),
582 )
583 .await
584 .expect("fetch should not hang");
585
586 handle.join().ok();
587
588 let output = result.expect("should succeed");
589 assert!(output.contains("Plain text content"));
590 }
591
592 #[tokio::test]
593 async fn web_fetch_does_not_follow_redirect_to_private_ip() {
594 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
603 let addr = listener.local_addr().unwrap();
604
605 let handle = std::thread::spawn(move || {
606 let (mut stream, _) = listener.accept().unwrap();
607 use std::io::{Read, Write};
608 let mut buf = [0u8; 4096];
609 let _ = stream.read(&mut buf);
610
611 let response = "HTTP/1.1 302 Found\r\nLocation: http://169.254.169.254/latest/meta-data/\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
612 write!(stream, "{}", response).unwrap();
613 stream.flush().unwrap();
614 });
615
616 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
617
618 let tool = WebFetch::new();
620 let result = tokio::time::timeout(
623 Duration::from_secs(10),
624 tool.fetch(&format!("http://{addr}/redirect"), DEFAULT_MAX_BYTES),
625 )
626 .await;
627
628 handle.join().ok();
629
630 let err_msg = match result {
631 Err(_elapsed) => panic!("fetch hung: redirect may have been followed"),
632 Ok(res) => match res {
633 Ok(body) => panic!("302 to a private IP must be an error, got body: {body}"),
634 Err(e) => e.to_string(),
635 },
636 };
637 assert!(
640 err_msg.contains("does not follow redirects"),
641 "unexpected error: {err_msg}"
642 );
643 assert!(err_msg.contains("302"));
644 assert!(err_msg.contains("169.254.169.254"));
645 }
646
647 #[tokio::test]
648 async fn web_fetch_rejects_invalid_url() {
649 let tool = WebFetch::new();
650 let result = tool
651 .execute(json!({ "url": "not-a-url" }))
652 .await
653 .unwrap_err();
654 assert!(result.to_string().contains("http:// or https://"));
655 }
656
657 #[tokio::test]
658 async fn web_fetch_rejects_missing_url() {
659 let tool = WebFetch::new();
660 let result = tool.execute(json!({})).await.unwrap_err();
661 assert!(result.to_string().contains("missing `url`"));
662 }
663
664 #[tokio::test]
665 async fn web_fetch_handles_html_content_type() {
666 let html = "<html><body><h1>Title</h1><p>Hello <a href=\"http://ex.com\">Link</a></p></body></html>";
668 let md = WebFetch::html_to_markdown(html);
669 assert!(md.contains("# Title"));
670 assert!(md.contains("Link"));
671 }
672
673 #[test]
674 fn truncate_multibyte_body_does_not_panic() {
675 let chinese_body = "你好世界".repeat(100);
678 let truncated = crate::truncate_str(&chinese_body, 100);
681 assert!(
682 truncated.len() <= 100,
683 "truncated len must not exceed max_bytes"
684 );
685 assert!(std::str::from_utf8(truncated.as_bytes()).is_ok());
687 }
688
689 #[test]
692 fn is_private_ip_ipv4_private_ranges() {
693 use std::net::IpAddr;
694 assert!(is_private_ip("10.0.0.1".parse::<IpAddr>().unwrap()));
696 assert!(is_private_ip("172.16.0.1".parse::<IpAddr>().unwrap()));
697 assert!(is_private_ip("192.168.1.1".parse::<IpAddr>().unwrap()));
698 assert!(is_private_ip("127.0.0.1".parse::<IpAddr>().unwrap()));
700 assert!(is_private_ip("169.254.169.254".parse::<IpAddr>().unwrap()));
702 assert!(!is_private_ip("8.8.8.8".parse::<IpAddr>().unwrap()));
704 assert!(!is_private_ip("93.184.216.34".parse::<IpAddr>().unwrap()));
705 }
706
707 #[test]
708 fn is_private_ip_ipv6_private_ranges() {
709 use std::net::IpAddr;
710 assert!(is_private_ip("::1".parse::<IpAddr>().unwrap()));
712 assert!(is_private_ip("::".parse::<IpAddr>().unwrap()));
714 assert!(is_private_ip("fc00::1".parse::<IpAddr>().unwrap()));
716 assert!(is_private_ip("fd00::1".parse::<IpAddr>().unwrap())); assert!(is_private_ip("fe80::1".parse::<IpAddr>().unwrap()));
719 assert!(!is_private_ip(
721 "2001:4860:4860::8888".parse::<IpAddr>().unwrap()
722 )); assert!(!is_private_ip(
724 "2606:4700:4700::1111".parse::<IpAddr>().unwrap()
725 )); }
727
728 #[test]
729 fn is_private_ip_each_ipv4_condition_independently() {
730 use std::net::IpAddr;
731 let loopback: IpAddr = "127.0.0.1".parse().unwrap();
734 let private: IpAddr = "192.168.0.1".parse().unwrap();
735 let link_local: IpAddr = "169.254.0.1".parse().unwrap();
736 let broadcast: IpAddr = "255.255.255.255".parse().unwrap();
737 let unspecified: IpAddr = "0.0.0.0".parse().unwrap();
738 assert!(is_private_ip(loopback), "loopback must be private");
739 assert!(is_private_ip(private), "private RFC1918 must be private");
740 assert!(is_private_ip(link_local), "link-local must be private");
741 assert!(is_private_ip(broadcast), "broadcast must be private");
742 assert!(is_private_ip(unspecified), "unspecified must be private");
743 }
744}