recursive-agent 0.8.2

A Rust coding-agent platform: kernel, tools, multi-agent orchestration, HTTP API, MCP, and TUI
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
//! `web_fetch`: HTTP GET tool for fetching web content.
//!
//! Supports plain text and HTML content with optional truncation.

use async_trait::async_trait;
use reqwest::Client;
use serde_json::{json, Value};
use std::time::Duration;

use super::Tool;
use crate::acp::ToolKind;
use crate::error::{Error, Result};
use crate::llm::ToolSpec;
use crate::tools::url_guard::validate_url;

const DEFAULT_MAX_BYTES: usize = 65536;
const REQUEST_TIMEOUT_SECS: u64 = 15;
const CONNECT_TIMEOUT_SECS: u64 = 5;

#[derive(Debug, Clone)]
pub struct WebFetch {
    client: Client,
}

impl Default for WebFetch {
    fn default() -> Self {
        Self::new()
    }
}

impl WebFetch {
    pub fn new() -> Self {
        let client = Client::builder()
            .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
            .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
            .user_agent(format!("recursive-agent/{}", env!("CARGO_PKG_VERSION")))
            // Do not follow redirects. `validate_url` (url_guard) checks only
            // the *initial* URL, so reqwest's default policy (follow up to 10
            // redirects) would let a public URL 302-redirect the fetch to a
            // private/internal host — an SSRF bypass. Under `Policy::none()`, a
            // 3xx comes back as a normal response and `fetch` surfaces it as an
            // error instead of issuing a second request. (Goal 372)
            .redirect(reqwest::redirect::Policy::none())
            // Construction carve-out: if TLS backend fails to initialize, the
            // process cannot perform HTTP requests at all. This is a fatal
            // startup condition equivalent to the providers.rs TOML parse
            // (Invariant #5 §construction).
            .build();
        #[allow(
            clippy::expect_used,
            reason = "TLS backend unavailable is a fatal startup error"
        )]
        let client = client.expect("reqwest client build: TLS backend unavailable");
        Self { client }
    }

    /// Perform the HTTP GET against a URL that has already passed
    /// [`validate_url`]. Kept separate from `execute` so tests can drive the
    /// HTTP path against a loopback mock without tripping the SSRF guard on
    /// the *initial* request (the guard itself is covered by
    /// `validate_url_blocks_ssrf_targets`).
    async fn fetch(&self, validated_url: &str, max_bytes: usize) -> Result<String> {
        let response = self
            .client
            .get(validated_url)
            .send()
            .await
            .map_err(|e| Error::Tool {
                name: "WebFetch".into(),
                call_id: None,
                message: format!("request failed: {}", e),
            })?;

        let status = response.status();
        // With `Policy::none()` a 3xx is a normal response, not an error — we
        // must surface it ourselves. Following it would bypass the SSRF guard
        // (the Location header can point at a private host).
        if status.is_redirection() {
            let location = response
                .headers()
                .get("location")
                .and_then(|v| v.to_str().ok())
                .unwrap_or("<none>");
            return Err(Error::Tool {
                name: "WebFetch".into(),
                call_id: None,
                message: format!(
                    "WebFetch does not follow redirects; got HTTP {} (Location: {})",
                    status.as_u16(),
                    location
                ),
            });
        }
        if !status.is_success() {
            let body = response.text().await.unwrap_or_default();
            let excerpt = if body.len() > 200 {
                format!("{}...", crate::truncate_str(&body, 200))
            } else {
                body
            };
            return Err(Error::Tool {
                name: "WebFetch".into(),
                call_id: None,
                message: format!("HTTP {}: {}", status.as_u16(), excerpt),
            });
        }

        let content_type = response
            .headers()
            .get("content-type")
            .and_then(|v| v.to_str().ok())
            .unwrap_or("")
            .to_string();

        let body = response.text().await.map_err(|e| Error::Tool {
            name: "WebFetch".into(),
            call_id: None,
            message: format!("failed to read response body: {}", e),
        })?;

        let total_bytes = body.len();
        let truncated = if total_bytes > max_bytes {
            let truncated_body = crate::truncate_str(&body, max_bytes);
            format!(
                "{}\n\n[…truncated at {} bytes; total body was {} bytes]",
                truncated_body, max_bytes, total_bytes
            )
        } else {
            body
        };

        // Convert HTML to markdown if content type suggests HTML
        if content_type.contains("text/html") {
            return Ok(Self::html_to_markdown(&truncated));
        }

        Ok(truncated)
    }

    /// Simple HTML to markdown conversion - handles links, headings, basic tags.
    fn html_to_markdown(html: &str) -> String {
        let mut result = String::new();
        let mut in_tag = false;
        let mut chars = html.chars().peekable();
        let mut link_href: Option<String> = None;
        let mut link_text = String::new();
        let mut in_link = false;

        while let Some(c) = chars.next() {
            if c == '<' {
                // Check for closing </a> tag before processing new tag
                if in_link {
                    // End of link - output as markdown
                    if let Some(href) = link_href.take() {
                        let txt = link_text.trim();
                        if !txt.is_empty() {
                            result.push_str(&format!("[{}]({})", txt, href));
                        }
                    }
                    link_text.clear();
                    in_link = false;
                }
                in_tag = true;
                // Look ahead to see what tag this is
                let mut tag_buf = String::new();
                let mut tag_chars = chars.clone();
                while let Some(&nc) = tag_chars.peek() {
                    if nc == '>' {
                        break;
                    }
                    tag_buf.push(nc);
                    tag_chars.next();
                }
                let tag_lower = tag_buf.to_lowercase();

                // Handle opening <a> tag
                if tag_lower.starts_with("a ") || tag_lower == "a>" {
                    in_link = true;
                    // Extract href if present - search within tag_buf for href="..."
                    if let Some(start) = tag_buf.find("href=\"") {
                        let url_start = start + 6; // skip href="
                                                   // Find the closing quote
                        if let Some(end_quote) = tag_buf[url_start..].find('"') {
                            link_href = Some(tag_buf[url_start..url_start + end_quote].to_string());
                        }
                    }
                }

                // Handle heading opening tags - output marker immediately
                if tag_lower.starts_with("h1") {
                    result.push_str("# ");
                } else if tag_lower.starts_with("h2") {
                    result.push_str("## ");
                } else if tag_lower.starts_with("h3") {
                    result.push_str("### ");
                }

                // Skip content in script/style tags
                if tag_lower.starts_with("script") || tag_lower.starts_with("style") {
                    // Skip until </script> or </style>
                    let close_tag = if tag_lower.starts_with("script") {
                        "</script>"
                    } else {
                        "</style>"
                    };
                    let remaining: String = chars.clone().collect();
                    if let Some(pos) = remaining.to_lowercase().find(close_tag) {
                        for _ in 0..pos + close_tag.len() {
                            chars.next();
                        }
                    }
                    in_tag = false;
                }
                continue;
            }

            if in_tag {
                if c == '>' {
                    in_tag = false;
                    // Check for block-level closing tags that add newlines
                    let remaining: String = chars.clone().take(10).collect();
                    let remaining_lower = remaining.to_lowercase();
                    if remaining_lower.starts_with("</p>")
                        || remaining_lower.starts_with("</div>")
                        || remaining_lower.starts_with("</li>")
                        || remaining_lower.starts_with("</h1>")
                        || remaining_lower.starts_with("</h2>")
                        || remaining_lower.starts_with("</h3>")
                    {
                        result.push('\n');
                    }
                    if remaining_lower.starts_with("<br") {
                        result.push('\n');
                    }
                }
                // Don't collect text while inside the tag - just skip it
                continue;
            }

            // Regular text - when in a link, collect for link; otherwise output
            if in_link {
                link_text.push(c);
            } else {
                if c.is_whitespace() {
                    if !result.ends_with(' ') && !result.ends_with('\n') {
                        result.push(' ');
                    }
                } else {
                    result.push(c);
                }
            }
        }

        // Handle any trailing link
        if in_link {
            if let Some(href) = link_href.take() {
                let txt = link_text.trim();
                if !txt.is_empty() {
                    result.push_str(&format!("[{}]({})", txt, href));
                }
            }
        }

        // Clean up: collapse multiple spaces, remove leading/trailing whitespace per line
        let lines: Vec<String> = result
            .lines()
            .map(|line| {
                let trimmed = line.trim();
                if trimmed.is_empty() {
                    String::new()
                } else {
                    // Collapse multiple spaces
                    let mut collapsed = String::new();
                    let mut last_was_space = false;
                    for c in trimmed.chars() {
                        if c.is_whitespace() {
                            if !last_was_space {
                                collapsed.push(' ');
                                last_was_space = true;
                            }
                        } else {
                            collapsed.push(c);
                            last_was_space = false;
                        }
                    }
                    collapsed
                }
            })
            .collect();

        lines.join("\n")
    }
}

#[async_trait]
impl Tool for WebFetch {
    fn spec(&self) -> ToolSpec {
        ToolSpec {
            name: "WebFetch".into(),
            description: "Fetch content from a URL via HTTP GET. Returns the body as text, optionally truncated to max_bytes. For HTML pages, attempts basic markdown conversion."
                .into(),
            parameters: json!({
                "type": "object",
                "properties": {
                    "url": { "type": "string", "description": "URL to fetch. Must start with http:// or https://." },
                    "max_bytes": { "type": "integer", "description": "Maximum bytes to read from body. Defaults to 65536. Truncation adds a note." }
                },
                "required": ["url"]
            }),
        }
    }

    fn is_deferred(&self) -> bool {
        true
    }

    fn side_effect_class(&self) -> crate::tools::ToolSideEffect {
        crate::tools::ToolSideEffect::ReadOnly
    }

    fn kind(&self) -> ToolKind {
        ToolKind::Fetch
    }

    async fn execute(&self, args: Value) -> Result<String> {
        let url = args["url"].as_str().ok_or_else(|| Error::BadToolArgs {
            name: "WebFetch".into(),
            message: "missing `url`".into(),
        })?;

        let validated_url = validate_url(url)?;

        let max_bytes = args
            .get("max_bytes")
            .and_then(|v| v.as_u64())
            .map(|n| n as usize)
            .unwrap_or(DEFAULT_MAX_BYTES);

        self.fetch(&validated_url, max_bytes).await
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::tools::url_guard::is_private_ip;

    #[test]
    fn validate_url_rejects_invalid() {
        assert!(validate_url("ftp://example.com").is_err());
        assert!(validate_url("example.com").is_err());
        assert!(validate_url("/path").is_err());
    }

    #[test]
    fn validate_url_accepts_valid() {
        assert!(validate_url("http://example.com").is_ok());
        assert!(validate_url("https://example.com").is_ok());
    }

    #[test]
    fn validate_url_blocks_ssrf_targets() {
        // localhost variants
        assert!(validate_url("http://localhost/secret").is_err());
        assert!(validate_url("http://localhost:8080/admin").is_err());
        // loopback IP
        assert!(validate_url("http://127.0.0.1/").is_err());
        assert!(validate_url("http://127.1.2.3/").is_err());
        // link-local / AWS IMDS
        assert!(validate_url("http://169.254.169.254/latest/meta-data/").is_err());
        // RFC-1918 private
        assert!(validate_url("http://10.0.0.1/").is_err());
        assert!(validate_url("http://192.168.1.1/").is_err());
        assert!(validate_url("http://172.16.0.1/").is_err());
        // IPv6 loopback
        assert!(validate_url("http://[::1]/").is_err());
        // GCP metadata
        assert!(validate_url("http://metadata.google.internal/").is_err());
    }

    #[test]
    fn html_to_markdown_strips_scripts() {
        let html = "<html><script>alert('x')</script><body>Hello</body></html>";
        let md = WebFetch::html_to_markdown(html);
        assert!(!md.contains("alert"));
        assert!(md.contains("Hello"));
    }

    #[test]
    fn html_to_markdown_preserves_links() {
        let html = "<a href=\"https://example.com\">Example</a>";
        let md = WebFetch::html_to_markdown(html);
        assert!(md.contains("[Example](https://example.com)"));
    }

    #[test]
    fn html_to_markdown_preserves_headings() {
        let html = "<h1>Title</h1><p>Para</p>";
        let md = WebFetch::html_to_markdown(html);
        assert!(md.contains("# Title"));
        assert!(md.contains("Para"));
    }

    #[test]
    fn html_to_markdown_collapse_whitespace() {
        let html = "<p>Hello    World</p>";
        let md = WebFetch::html_to_markdown(html);
        assert!(md.contains("Hello World"));
    }

    #[test]
    fn collapse_whitespace_basic() {
        // Test the internal behavior by checking result
        let html = "Hello   World";
        let md = WebFetch::html_to_markdown(html);
        assert!(md.contains("Hello World"));
    }

    #[test]
    fn web_fetch_construction_smoke() {
        let tool = WebFetch::new();
        assert_eq!(tool.spec().name, "WebFetch");
    }

    #[tokio::test]
    async fn test_a_mock_server_returns_text_plain() {
        // Spawn mock server
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();

        let handle = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().unwrap();
            use std::io::{Read, Write};
            let mut buf = [0u8; 4096];
            let _ = stream.read(&mut buf);

            let body = "Hello, world!";
            let response = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
                body.len(),
                body
            );
            write!(stream, "{}", response).unwrap();
            stream.flush().unwrap();
        });

        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

        let client = Client::builder()
            .timeout(Duration::from_secs(2))
            .connect_timeout(Duration::from_secs(1))
            .build()
            .unwrap();

        let result = client.get(format!("http://{addr}")).send().await;

        handle.join().ok();

        assert!(result.is_ok());
        let resp = result.unwrap();
        assert!(resp.status().is_success());
        let body = resp.text().await.unwrap();
        assert_eq!(body, "Hello, world!");
    }

    #[tokio::test]
    async fn test_b_mock_server_returns_404() {
        // Spawn mock server returning 404
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();

        let handle = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().unwrap();
            use std::io::{Read, Write};
            let mut buf = [0u8; 4096];
            let _ = stream.read(&mut buf);

            let body = "Not Found";
            let response = format!(
                "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
                body.len(),
                body
            );
            write!(stream, "{}", response).unwrap();
            stream.flush().unwrap();
        });

        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

        let client = Client::builder()
            .timeout(Duration::from_secs(2))
            .connect_timeout(Duration::from_secs(1))
            .build()
            .unwrap();

        let result = client.get(format!("http://{addr}")).send().await;

        handle.join().ok();

        assert!(result.is_ok());
        let resp = result.unwrap();
        assert_eq!(resp.status().as_u16(), 404);
    }

    #[tokio::test]
    async fn test_c_body_exceeds_max_bytes() {
        // Spawn mock server returning large body
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();

        let body = "a".repeat(200);
        let handle = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().unwrap();
            use std::io::{Read, Write};
            let mut buf = [0u8; 4096];
            let _ = stream.read(&mut buf);

            let response = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
                body.len(),
                body
            );
            write!(stream, "{}", response).unwrap();
            stream.flush().unwrap();
        });

        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

        let tool = WebFetch::new();
        // Drive `fetch` directly (not `execute`): validate_url would reject the
        // 127.0.0.1 mock URL before any request is made. The SSRF guard on the
        // initial URL is covered by `validate_url_blocks_ssrf_targets`.
        let result = tokio::time::timeout(
            Duration::from_secs(10),
            tool.fetch(&format!("http://{addr}/test"), 50),
        )
        .await
        .expect("fetch should not hang");

        handle.join().ok();

        let output = result.expect("should succeed");
        // Should contain truncated body
        assert!(output.contains("aaaaaaaa"));
        // Should contain truncation marker
        assert!(output.contains("truncated"));
        // Should mention original size
        assert!(output.contains("200 bytes"));
    }

    #[tokio::test]
    async fn web_fetch_tool_on_mock_server() {
        // Test the full tool with mock server
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();

        let body = "Plain text content";
        let handle = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().unwrap();
            use std::io::{Read, Write};
            let mut buf = [0u8; 4096];
            let _ = stream.read(&mut buf);

            let response = format!(
                "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
                body.len(),
                body
            );
            write!(stream, "{}", response).unwrap();
            stream.flush().unwrap();
        });

        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

        let tool = WebFetch::new();
        // Drive `fetch` directly (not `execute`): validate_url would reject the
        // 127.0.0.1 mock URL before any request is made. The SSRF guard on the
        // initial URL is covered by `validate_url_blocks_ssrf_targets`.
        let result = tokio::time::timeout(
            Duration::from_secs(10),
            tool.fetch(&format!("http://{addr}/test"), DEFAULT_MAX_BYTES),
        )
        .await
        .expect("fetch should not hang");

        handle.join().ok();

        let output = result.expect("should succeed");
        assert!(output.contains("Plain text content"));
    }

    #[tokio::test]
    async fn web_fetch_does_not_follow_redirect_to_private_ip() {
        // Regression for the SSRF redirect bypass (Goal 372): a URL that 302s
        // to a private host (AWS IMDS) must NOT be followed. The production
        // client (WebFetch::new) is built with redirect(Policy::none()), so a
        // 3xx is surfaced as an error instead of a second request to
        // 169.254.169.254. We drive `fetch` directly so the SSRF guard on the
        // *initial* 127.0.0.1 mock URL doesn't block the request before the
        // redirect logic is reached; the guard itself is covered by
        // `validate_url_blocks_ssrf_targets`.
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();

        let handle = std::thread::spawn(move || {
            let (mut stream, _) = listener.accept().unwrap();
            use std::io::{Read, Write};
            let mut buf = [0u8; 4096];
            let _ = stream.read(&mut buf);

            let response = "HTTP/1.1 302 Found\r\nLocation: http://169.254.169.254/latest/meta-data/\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
            write!(stream, "{}", response).unwrap();
            stream.flush().unwrap();
        });

        tokio::time::sleep(std::time::Duration::from_millis(50)).await;

        // Production tool: exercises the Policy::none() client built by new().
        let tool = WebFetch::new();
        // Timeout-guard the fetch so a regression (redirect followed to an
        // unroutable/IMDS host) can't hang the suite.
        let result = tokio::time::timeout(
            Duration::from_secs(10),
            tool.fetch(&format!("http://{addr}/redirect"), DEFAULT_MAX_BYTES),
        )
        .await;

        handle.join().ok();

        let err_msg = match result {
            Err(_elapsed) => panic!("fetch hung: redirect may have been followed"),
            Ok(res) => match res {
                Ok(body) => panic!("302 to a private IP must be an error, got body: {body}"),
                Err(e) => e.to_string(),
            },
        };
        // Must be the "does not follow redirects" 3xx error — NOT a successful
        // body read from 169.254.169.254 and NOT a connection error to it.
        assert!(
            err_msg.contains("does not follow redirects"),
            "unexpected error: {err_msg}"
        );
        assert!(err_msg.contains("302"));
        assert!(err_msg.contains("169.254.169.254"));
    }

    #[tokio::test]
    async fn web_fetch_rejects_invalid_url() {
        let tool = WebFetch::new();
        let result = tool
            .execute(json!({ "url": "not-a-url" }))
            .await
            .unwrap_err();
        assert!(result.to_string().contains("http:// or https://"));
    }

    #[tokio::test]
    async fn web_fetch_rejects_missing_url() {
        let tool = WebFetch::new();
        let result = tool.execute(json!({})).await.unwrap_err();
        assert!(result.to_string().contains("missing `url`"));
    }

    #[tokio::test]
    async fn web_fetch_handles_html_content_type() {
        // Test HTML to markdown conversion
        let html = "<html><body><h1>Title</h1><p>Hello <a href=\"http://ex.com\">Link</a></p></body></html>";
        let md = WebFetch::html_to_markdown(html);
        assert!(md.contains("# Title"));
        assert!(md.contains("Link"));
    }

    #[test]
    fn truncate_multibyte_body_does_not_panic() {
        // "你好世界" = 4 chars × 3 bytes each = 12 bytes per repeat.
        // 100 repeats = 1200 bytes total.
        let chinese_body = "你好世界".repeat(100);
        // max_bytes=100 would cut through a 3-byte sequence with naive byte indexing.
        // crate::truncate_str must return a valid &str without panicking.
        let truncated = crate::truncate_str(&chinese_body, 100);
        assert!(
            truncated.len() <= 100,
            "truncated len must not exceed max_bytes"
        );
        // The result must be valid UTF-8 (would panic at assert if not, catching any regression).
        assert!(std::str::from_utf8(truncated.as_bytes()).is_ok());
    }

    // ── is_private_ip direct tests ───────────────────────────────────────────

    #[test]
    fn is_private_ip_ipv4_private_ranges() {
        use std::net::IpAddr;
        // RFC-1918 private ranges
        assert!(is_private_ip("10.0.0.1".parse::<IpAddr>().unwrap()));
        assert!(is_private_ip("172.16.0.1".parse::<IpAddr>().unwrap()));
        assert!(is_private_ip("192.168.1.1".parse::<IpAddr>().unwrap()));
        // Loopback
        assert!(is_private_ip("127.0.0.1".parse::<IpAddr>().unwrap()));
        // Link-local (AWS IMDS)
        assert!(is_private_ip("169.254.169.254".parse::<IpAddr>().unwrap()));
        // Public IP should NOT be private
        assert!(!is_private_ip("8.8.8.8".parse::<IpAddr>().unwrap()));
        assert!(!is_private_ip("93.184.216.34".parse::<IpAddr>().unwrap()));
    }

    #[test]
    fn is_private_ip_ipv6_private_ranges() {
        use std::net::IpAddr;
        // IPv6 loopback (::1)
        assert!(is_private_ip("::1".parse::<IpAddr>().unwrap()));
        // IPv6 unspecified (::)
        assert!(is_private_ip("::".parse::<IpAddr>().unwrap()));
        // fc00::/7 ULA private (kills & → | and & → ^ mutants at segment mask check)
        assert!(is_private_ip("fc00::1".parse::<IpAddr>().unwrap()));
        assert!(is_private_ip("fd00::1".parse::<IpAddr>().unwrap())); // fd is also in fc00::/7
                                                                      // fe80::/10 link-local (kills & → | and & → ^ mutants)
        assert!(is_private_ip("fe80::1".parse::<IpAddr>().unwrap()));
        // Public IPv6 should NOT be private
        assert!(!is_private_ip(
            "2001:4860:4860::8888".parse::<IpAddr>().unwrap()
        )); // Google DNS
        assert!(!is_private_ip(
            "2606:4700:4700::1111".parse::<IpAddr>().unwrap()
        )); // Cloudflare
    }

    #[test]
    fn is_private_ip_each_ipv4_condition_independently() {
        use std::net::IpAddr;
        // Ensure each individual || branch is tested
        // (kills || → && mutants)
        let loopback: IpAddr = "127.0.0.1".parse().unwrap();
        let private: IpAddr = "192.168.0.1".parse().unwrap();
        let link_local: IpAddr = "169.254.0.1".parse().unwrap();
        let broadcast: IpAddr = "255.255.255.255".parse().unwrap();
        let unspecified: IpAddr = "0.0.0.0".parse().unwrap();
        assert!(is_private_ip(loopback), "loopback must be private");
        assert!(is_private_ip(private), "private RFC1918 must be private");
        assert!(is_private_ip(link_local), "link-local must be private");
        assert!(is_private_ip(broadcast), "broadcast must be private");
        assert!(is_private_ip(unspecified), "unspecified must be private");
    }
}