vtcode_core/tools/web_fetch/
mod.rs

1//! WebFetch tool for fetching and analyzing web content using AI
2//!
3//! Supports both restricted (blocklist) and whitelist (allowlist) modes
4//! with dynamic configuration loading from vtcode.toml
5//!
6//! Fetched content is written to ephemeral cache files under the user cache
7//! directory's `web-fetch/` path.
8//! and cleaned up periodically. Content is never persisted to the workspace.
9
10use super::traits::Tool;
11use crate::config::constants::tools;
12use anyhow::{Context, Result, anyhow, bail};
13use async_trait::async_trait;
14use futures::stream::StreamExt;
15use hashbrown::HashSet;
16use reqwest::header::{ACCEPT, HeaderMap, HeaderValue, USER_AGENT};
17use serde::Deserialize;
18use serde_json::{Value, json};
19use std::path::PathBuf;
20use url::Url;
21use vtcode_commons::VtCodePaths;
22
23pub mod classify_helpers;
24pub mod domains;
25mod errors;
26mod html_preview;
27mod network_guards;
28pub use classify_helpers::extract_http_status;
29pub use domains::{BUILTIN_BLOCKED_DOMAINS, BUILTIN_BLOCKED_PATTERNS, MALICIOUS_PATTERNS};
30use errors::web_fetch_error_response;
31pub(super) use network_guards::is_private_host;
32use network_guards::{domain_matches_allowed, extract_domain};
33
34const DEFAULT_TIMEOUT_SECS: u64 = 30;
35const MAX_CONTENT_SIZE: usize = 500_000; // 500KB max content size
36const MAX_ALLOWED_BYTES: usize = 2_000_000; // 2MB hard cap
37const MAX_ALLOWED_TIMEOUT_SECS: u64 = 120; // 2 minutes hard cap
38
39/// Subdirectory under the user cache directory for ephemeral web-fetch artifacts.
40const TEMP_SUBDIR: &str = "web_fetch";
41
42/// Max age in seconds before temp files are cleaned up (1 hour).
43const TEMP_MAX_AGE_SECS: u64 = 3600;
44
45pub(crate) const WEB_FETCH_DESCRIPTION: &str = "Fetch a remote URL and return the start of the page inline. The result contains `preview` (readable text for HTML pages, else the first 8000 bytes of the body), `preview_kind` (`text` or `raw`), `content_length`, and a `temp_file` path holding the full fetched body; read `temp_file` only when the preview is not enough. The tool does not analyze the page: `prompt` is returned with the result to guide your own reading. Set format=markdown to get cleaned markdown inline from the defuddle.md extraction service instead; that mode allows 1 call per session. For docs domains, try /llms.txt first: for 'abc.com', fetch https://abc.com/llms.txt before the homepage, then traverse linked URLs for relevant Markdown sources. Prefer llms.txt over llms-full.txt, which can be multi-megabyte. A body larger than max_bytes is cut off and the result reports truncated_by_max_bytes and source_size_bytes, so you can retry with a larger max_bytes. Temp files are ephemeral and may be cleaned up after about an hour.";
46
47/// Parameter schema for `web_fetch`, shared by the distributed builtin
48/// registration and the web tool pack so the two surfaces cannot drift.
49///
50/// `format` must be declared here: argument validation rejects unknown
51/// properties (`additionalProperties: false`), so an undeclared `format`
52/// would make the markdown route in [`WebFetchTool::execute`] unreachable.
53pub(crate) fn web_fetch_parameter_schema() -> Value {
54    json!({
55        "type": "object",
56        "properties": {
57            "url": {
58                "type": "string",
59                "description": "URL to fetch (HTTPS required by default)"
60            },
61            "prompt": {
62                "type": "string",
63                "description": "Question or instruction describing what to look for in the fetched content. It is returned with the result to guide your analysis; omit it to get a default summarize instruction."
64            },
65            "format": {
66                "type": "string",
67                "enum": ["summary", "markdown"],
68                "description": "Output mode (default: summary). summary fetches the page directly and returns a preview plus a temp_file path. markdown returns the page as cleaned markdown inline via the defuddle.md extraction service; it allows 1 call per session, accepts only remote http(s) URLs, caps max_bytes at 262144, and ignores prompt and timeout_secs."
69            },
70            "max_bytes": {
71                "type": "integer",
72                "description": "Maximum response body size in bytes (default: 500000, max: 2000000). The default fits most pages, including llms.txt, so set this only to raise the limit after a truncated_by_max_bytes result or to cap a very large page."
73            },
74            "timeout_secs": {
75                "type": "integer",
76                "description": "Request timeout in seconds (default: 30, max: 120)"
77            }
78        },
79        "required": ["url"],
80        "additionalProperties": false
81    })
82}
83
84#[derive(Debug, Deserialize)]
85struct WebFetchArgs {
86    url: String,
87    prompt: String,
88    #[serde(default)]
89    max_bytes: Option<usize>,
90    #[serde(default)]
91    timeout_secs: Option<u64>,
92}
93
94/// WebFetch tool that fetches URL content and processes it with AI
95#[derive(Clone)]
96pub struct WebFetchTool {
97    /// Security mode: restricted (blocklist) or whitelist (allowlist)
98    pub mode: vtcode_config::WebFetchMode,
99    /// Additional blocked domains (merged with builtin)
100    pub blocked_domains: HashSet<String>,
101    /// Additional blocked patterns (merged with builtin)
102    pub blocked_patterns: Vec<String>,
103    /// Allowed domains (for exemptions in restricted mode or primary list in whitelist mode)
104    pub allowed_domains: HashSet<String>,
105    /// Defuddle markdown-extraction backend (used when `format: "markdown"`).
106    /// Shared session-cap state survives clones (Arc counter inside).
107    pub defuddle: crate::tools::defuddle::DefuddleTool,
108    /// Strict HTTPS-only mode
109    pub strict_https_only: bool,
110}
111
112struct FetchedWebContent {
113    content: String,
114    truncated_by_max_bytes: bool,
115    source_size_bytes: usize,
116}
117
118fn fetched_content_from_bytes(bytes: &[u8], max_bytes: usize) -> Result<FetchedWebContent> {
119    let source_size_bytes = bytes.len();
120    let truncated_by_max_bytes = source_size_bytes > max_bytes;
121    if !truncated_by_max_bytes {
122        return Ok(FetchedWebContent {
123            content: String::from_utf8(bytes.to_vec()).context("Response body is not valid UTF-8")?,
124            truncated_by_max_bytes,
125            source_size_bytes,
126        });
127    }
128
129    // Truncate to `max_bytes`, then back off to the nearest valid UTF-8
130    // boundary in a single O(n) pass via `valid_up_to`. The previous per-byte
131    // `end -= 1` loop was O(n^2) on adversarial bodies (e.g. a multi-MB
132    // document whose first max_bytes bytes are not UTF-8-aligned).
133    let valid_end = match std::str::from_utf8(&bytes[..max_bytes]) {
134        Ok(_) => max_bytes,
135        Err(err) => err.valid_up_to(),
136    };
137
138    let content = String::from_utf8(bytes[..valid_end].to_vec()).context("Response body is not valid UTF-8")?;
139    Ok(FetchedWebContent { content, truncated_by_max_bytes, source_size_bytes })
140}
141
142/// Returns the path to the ephemeral temp directory for web_fetch artifacts.
143/// Creates the directory if it doesn't exist.
144async fn web_fetch_temp_dir() -> Result<PathBuf> {
145    VtCodePaths::resolve()
146        .context("Could not resolve VT Code cache directory")?
147        .ensure_cache_child_dir(TEMP_SUBDIR)
148        .context("Failed to create web-fetch cache directory")
149}
150
151/// Write content to an ephemeral temp file and return its path.
152/// Files are named with a timestamp for easy age-based cleanup.
153async fn write_to_temp_file(content: &str, url: &str) -> Result<PathBuf> {
154    let temp_dir = web_fetch_temp_dir().await?;
155    let timestamp = std::time::SystemTime::now()
156        .duration_since(std::time::UNIX_EPOCH)
157        .unwrap_or_default()
158        .as_micros();
159
160    // Create a safe filename from the URL
161    let url_hash = {
162        use std::collections::hash_map::DefaultHasher;
163        use std::hash::{Hash, Hasher};
164        let mut hasher = DefaultHasher::new();
165        url.hash(&mut hasher);
166        format!("{:016x}", hasher.finish())
167    };
168
169    let filename = format!("{url_hash}_{timestamp}.txt");
170    let file_path = temp_dir.join(&filename);
171
172    let bytes = content.as_bytes().to_vec();
173    let write_path = file_path.clone();
174    tokio::task::spawn_blocking(move || VtCodePaths::write_private_file_atomic(&write_path, &bytes))
175        .await
176        .context("web-fetch cache writer task panicked")?
177        .with_context(|| format!("Failed to write temp file: {}", file_path.display()))?;
178
179    Ok(file_path)
180}
181
182/// Clean up old web_fetch temp files.
183pub async fn cleanup_old_web_fetch_temps(max_age_secs: u64) -> Result<usize> {
184    let temp_dir = match web_fetch_temp_dir().await {
185        Ok(d) => d,
186        Err(_) => return Ok(0),
187    };
188
189    if tokio::fs::metadata(&temp_dir).await.is_err() {
190        return Ok(0);
191    }
192
193    let cutoff = std::time::SystemTime::now()
194        .checked_sub(std::time::Duration::from_secs(max_age_secs))
195        .unwrap_or(std::time::UNIX_EPOCH);
196
197    let mut removed = 0;
198    let mut entries = match tokio::fs::read_dir(&temp_dir).await {
199        Ok(e) => e,
200        Err(_) => return Ok(0),
201    };
202
203    while let Some(entry) = entries.next_entry().await.unwrap_or(None) {
204        let path = entry.path();
205        if !path.is_file() {
206            continue;
207        }
208        if let Ok(metadata) = entry.metadata().await
209            && let Ok(modified) = metadata.modified()
210            && modified <= cutoff
211            && tokio::fs::remove_file(&path).await.is_ok()
212        {
213            removed += 1;
214        }
215    }
216
217    if removed > 0 {
218        tracing::info!(count = removed, "Cleaned up old web_fetch temp files");
219    }
220
221    Ok(removed)
222}
223
224impl WebFetchTool {
225    pub fn new() -> Self {
226        Self {
227            mode: vtcode_config::WebFetchMode::Restricted,
228            blocked_domains: HashSet::new(),
229            blocked_patterns: Vec::new(),
230            allowed_domains: HashSet::new(),
231            defuddle: crate::tools::defuddle::DefuddleTool::new(),
232            strict_https_only: true,
233        }
234    }
235
236    /// Create a WebFetchTool with custom configuration
237    pub fn with_config(
238        mode: vtcode_config::WebFetchMode,
239        blocked_domains: Vec<String>,
240        blocked_patterns: Vec<String>,
241        allowed_domains: Vec<String>,
242        strict_https_only: bool,
243    ) -> Self {
244        Self {
245            mode,
246            blocked_domains: blocked_domains.into_iter().collect(),
247            blocked_patterns,
248            allowed_domains: allowed_domains.into_iter().collect(),
249            defuddle: crate::tools::defuddle::DefuddleTool::new(),
250            strict_https_only,
251        }
252    }
253
254    /// Build a `WebFetchTool` from a `WebFetchConfig` value-object. This is
255    /// the entry point the tool registry uses so user-configured
256    /// allow/block lists and HTTPS settings actually take effect.
257    pub fn from_config(config: &vtcode_config::WebFetchConfig) -> Self {
258        Self::with_config(
259            config.mode,
260            config.blocked_domains.clone(),
261            config.blocked_patterns.clone(),
262            config.allowed_domains.clone(),
263            config.strict_https_only,
264        )
265    }
266
267    async fn fetch_url_content(&self, url: &str, max_bytes: usize, timeout_secs: u64) -> Result<FetchedWebContent> {
268        // Validate URL
269        self.validate_url(url)?;
270
271        let default_headers = Self::default_headers();
272
273        // Build a redirect policy that validates each redirect target against SSRF rules.
274        // Limits to 5 redirects to bound the chain and re-checks security on each hop.
275        let blocked_domains = self.blocked_domains.clone();
276        let blocked_patterns = self.blocked_patterns.clone();
277        let allowed_domains = self.allowed_domains.clone();
278        let strict_https = self.strict_https_only;
279        let redirect_policy = reqwest::redirect::Policy::custom(move |attempt| {
280            if attempt.previous().len() >= 5 {
281                return attempt.stop();
282            }
283            let next_url = attempt.url();
284            let next_str = next_url.as_str();
285
286            // Re-validate HTTPS if strict mode is on
287            if strict_https && !next_str.starts_with("https://") {
288                return attempt.stop();
289            }
290
291            // Re-validate the redirect target's domain against the same security checks
292            if let Ok(domain) = extract_domain(next_str) {
293                if is_private_host(&domain) {
294                    return attempt.stop();
295                }
296                let domain_lower = domain.to_ascii_lowercase();
297                if domain_lower.ends_with(".local")
298                    || domain_lower.ends_with(".internal")
299                    || domain_lower.ends_with(".localhost")
300                    || domain_lower.ends_with(".test")
301                    || domain_lower.ends_with(".invalid")
302                    || domain_lower.ends_with(".home.arpa")
303                {
304                    return attempt.stop();
305                }
306                // Check blocked domains
307                let mut all_blocked = BUILTIN_BLOCKED_DOMAINS.to_vec();
308                all_blocked.extend(blocked_domains.iter().map(|s| s.as_str()));
309                for blocked in &all_blocked {
310                    if next_str.to_lowercase().contains(blocked) {
311                        return attempt.stop();
312                    }
313                }
314                // Check blocked patterns
315                let mut all_patterns = BUILTIN_BLOCKED_PATTERNS.to_vec();
316                all_patterns.extend(blocked_patterns.iter().map(|s| s.as_str()));
317                for pattern in &all_patterns {
318                    if next_str.to_lowercase().contains(pattern) {
319                        return attempt.stop();
320                    }
321                }
322                // Check allowed domains (exemptions)
323                for allowed in &allowed_domains {
324                    if domain_matches_allowed(&domain, allowed) {
325                        return attempt.follow();
326                    }
327                }
328            }
329            attempt.follow()
330        });
331
332        let client = reqwest::Client::builder()
333            .default_headers(default_headers)
334            .timeout(std::time::Duration::from_secs(timeout_secs))
335            .redirect(redirect_policy)
336            .build()?;
337
338        let response = client.get(url).send().await?;
339
340        if !response.status().is_success() {
341            return Err(anyhow!("HTTP request failed with status: {}", response.status()));
342        }
343
344        let content_type = response
345            .headers()
346            .get("content-type")
347            .and_then(|h| h.to_str().ok())
348            .unwrap_or("")
349            .to_string();
350
351        // Validate content type
352        self.validate_content_type(&content_type)?;
353
354        // Limit response body to max_bytes while still returning useful partial
355        // text. Oversized documentation pages are common, and a truncated fetch
356        // gives the agent more signal than an error-only result.
357        //
358        // Reject up-front when Content-Length already exceeds the cap, and
359        // stream `chunk()`s capped at max_bytes instead of buffering the entire
360        // body with `response.bytes()`. This bounds peak memory even for
361        // hostile servers that lie about or omit Content-Length.
362        if let Some(content_length) = response.content_length()
363            && content_length > max_bytes as u64
364        {
365            return Err(anyhow!(
366                "Response Content-Length {} exceeds max_bytes {}; refusing to buffer the full body",
367                content_length,
368                max_bytes
369            ));
370        }
371
372        let mut bytes = Vec::with_capacity(max_bytes.min(64 * 1024));
373        let mut stream = response.bytes_stream();
374        let mut max_reached = false;
375        while let Some(chunk) = stream.next().await {
376            let chunk = chunk.context("Failed to read response body")?;
377            if bytes.len() >= max_bytes {
378                max_reached = true;
379                break;
380            }
381            let take = chunk.len().min(max_bytes - bytes.len());
382            bytes.extend_from_slice(&chunk[..take]);
383        }
384
385        let mut fetched = fetched_content_from_bytes(&bytes, max_bytes)?;
386        // The stream stopped exactly at the cap, so the true body is at least
387        // max_bytes. `fetched_content_from_bytes` compares against `bytes.len()`
388        // and would report "not truncated" for a body that ends precisely at the
389        // cap — override so callers still see the truncation metadata.
390        if max_reached || bytes.len() >= max_bytes {
391            fetched.truncated_by_max_bytes = true;
392        }
393        Ok(fetched)
394    }
395
396    fn validate_url(&self, url: &str) -> Result<()> {
397        let parsed = Url::parse(url).map_err(|error| {
398            anyhow!(
399                "web_fetch only accepts remote HTTP(S) URLs; use read_file or unified_file to read local files: {error}"
400            )
401        })?;
402        if !matches!(parsed.scheme(), "http" | "https") {
403            bail!("web_fetch only accepts remote HTTP(S) URLs; use read_file or unified_file to read local files");
404        }
405
406        // HTTPS enforcement (can be disabled only for testing)
407        if self.strict_https_only && parsed.scheme() != "https" {
408            return Err(anyhow!("Only HTTPS URLs are allowed for security"));
409        }
410
411        // Parse the URL to extract the real host, which correctly separates
412        // userinfo credentials (http://evil@127.0.0.1/) from the host.
413        let domain = extract_domain(url).map_err(|e| anyhow!("Failed to parse URL for security validation: {e}"))?;
414
415        // Reject private, loopback, link-local, and reserved IPs.
416        if is_private_host(&domain) {
417            return Err(anyhow!("Access to local/private networks is blocked"));
418        }
419
420        // Block special-use TLDs per RFC 6761 and mDNS/split-DNS conventions:
421        // - .local (mDNS)
422        // - .internal (private split-DNS)
423        // - .localhost (loopback)
424        // - .test (testing, RFC 6761)
425        // - .invalid (always fails, RFC 6761)
426        // - .home.arpa (home networking, RFC 7788)
427        let domain_lower = domain.to_ascii_lowercase();
428        if domain_lower.ends_with(".local")
429            || domain_lower.ends_with(".internal")
430            || domain_lower.ends_with(".localhost")
431            || domain_lower.ends_with(".test")
432            || domain_lower.ends_with(".invalid")
433            || domain_lower.ends_with(".home.arpa")
434        {
435            return Err(anyhow!("Access to local/private networks is blocked"));
436        }
437
438        let url_lower = url.to_lowercase();
439
440        // Apply security policy based on mode
441        match self.mode {
442            vtcode_config::WebFetchMode::Whitelist => self.validate_whitelist_mode(&url_lower)?,
443            vtcode_config::WebFetchMode::Restricted => self.validate_restricted_mode(&url_lower)?,
444        }
445
446        Ok(())
447    }
448
449    fn validate_whitelist_mode(&self, url: &str) -> Result<()> {
450        // In whitelist mode, only explicitly allowed domains are permitted
451        let domain = extract_domain(url)?;
452
453        if self.allowed_domains.is_empty() {
454            return Err(anyhow!(
455                "Whitelist mode enabled but no domains are whitelisted. Configure allowed_domains in web_fetch settings."
456            ));
457        }
458
459        // Check if domain matches any whitelisted domain or pattern
460        for allowed in &self.allowed_domains {
461            if domain_matches_allowed(&domain, allowed) {
462                return Ok(());
463            }
464        }
465
466        Err(anyhow!(
467            "Domain '{domain}' is not in the whitelist. Only explicitly allowed domains are permitted in whitelist mode."
468        ))
469    }
470
471    fn validate_restricted_mode(&self, url: &str) -> Result<()> {
472        // In restricted mode, use a blocklist of known dangerous/sensitive domains
473        let url_lower = url.to_lowercase();
474
475        // Check against allowed exemptions first (exemptions override blocklist)
476        let domain = extract_domain(url)?;
477        for allowed in &self.allowed_domains {
478            if domain_matches_allowed(&domain, allowed) {
479                return Ok(());
480            }
481        }
482
483        // Check for malicious and sensitive URL patterns
484        self.validate_url_safety(&url_lower)?;
485
486        Ok(())
487    }
488
489    fn validate_url_safety(&self, url: &str) -> Result<()> {
490        // Combine built-in and custom blocked domains
491        let mut all_blocked_domains = BUILTIN_BLOCKED_DOMAINS.to_vec();
492        all_blocked_domains.extend(self.blocked_domains.iter().map(|s| s.as_str()));
493
494        // Combine built-in and custom blocked patterns
495        let mut all_blocked_patterns = BUILTIN_BLOCKED_PATTERNS.to_vec();
496        all_blocked_patterns.extend(self.blocked_patterns.iter().map(|s| s.as_str()));
497
498        // Check blocked domains
499        for domain in &all_blocked_domains {
500            if url.contains(domain) {
501                return Err(anyhow!(
502                    "Access to sensitive domain '{domain}' is blocked for privacy and security reasons"
503                ));
504            }
505        }
506
507        // Check for sensitive patterns in URL
508        for pattern in &all_blocked_patterns {
509            if url.contains(pattern) {
510                return Err(anyhow!(
511                    "URL contains sensitive pattern '{pattern}'. Fetching URLs with credentials or sensitive data is blocked"
512                ));
513            }
514        }
515
516        // Check for common malware/phishing indicators
517        self.check_malicious_indicators(url)?;
518
519        Ok(())
520    }
521
522    fn check_malicious_indicators(&self, url: &str) -> Result<()> {
523        for pattern in MALICIOUS_PATTERNS {
524            if url.contains(pattern) {
525                return Err(anyhow!("URL contains potentially malicious pattern. Access blocked for safety"));
526            }
527        }
528
529        Ok(())
530    }
531
532    fn validate_content_type(&self, content_type: &str) -> Result<()> {
533        if content_type.is_empty() {
534            return Ok(());
535        }
536
537        let allowed_types = [
538            "text/html",
539            "text/plain",
540            "text/markdown",
541            "application/json",
542            "application/xml",
543            "text/xml",
544            "application/javascript",
545            "text/css",
546            "text/javascript",
547            "application/xhtml+xml",
548        ];
549
550        // Content-Type headers may include parameters (e.g., "text/html; charset=utf-8").
551        // Extract just the media type (everything before the first ';') for matching.
552        let content_type_lower = content_type.to_lowercase();
553        let media_type = content_type_lower.split(';').next().unwrap_or(&content_type_lower).trim();
554
555        if allowed_types.contains(&media_type) {
556            Ok(())
557        } else {
558            Err(anyhow!(
559                "Content type '{content_type}' is not supported. Only text-based content types are allowed."
560            ))
561        }
562    }
563
564    async fn run(&self, raw_args: Value) -> Result<Value> {
565        let args: WebFetchArgs = serde_json::from_value(raw_args)
566            .context("Invalid arguments for web_fetch tool. Provide 'url' (and optionally 'prompt').")?;
567
568        let max_bytes = args.max_bytes.map(|v| v.min(MAX_ALLOWED_BYTES)).unwrap_or(MAX_CONTENT_SIZE);
569        let timeout_secs = args
570            .timeout_secs
571            .map(|v| v.min(MAX_ALLOWED_TIMEOUT_SECS))
572            .unwrap_or(DEFAULT_TIMEOUT_SECS);
573
574        // Fetch the URL content with detailed error handling
575        let fetched = match self.fetch_url_content(&args.url, max_bytes, timeout_secs).await {
576            Ok(fetched) => fetched,
577            Err(e) => {
578                // Categorize the error so the agent loop can react. We used to
579                // surface a flat string here, which led to the agent in
580                // turn_578 reporting "github.com, npmjs.com, and crates.io are
581                // all blocked" when the actual errors were 403/404 from the
582                // upstream services.
583                return Ok(web_fetch_error_response(&args.url, max_bytes, timeout_secs, &e));
584            }
585        };
586        let content = fetched.content;
587
588        let content_length = content.len();
589
590        if content_length == 0 {
591            return Ok(json!({
592                "error": format!(
593                    "web_fetch: no content fetched from '{}'. The URL may be unreachable, returned empty content, or used an unsupported content-type.",
594                    args.url
595                ),
596                "url": args.url,
597                "error_type": "empty_content",
598                "next_action": "The host returned an empty body. This may be a bot block, a JS-only page, or a real empty document. Try web_search to confirm the page exists; if it does, the content is probably JavaScript-rendered and you need a different tool."
599            }));
600        }
601
602        // Write content to ephemeral temp file instead of keeping it in the response.
603        // The temp file is auto-cleaned by age-based cleanup.
604        let temp_path = write_to_temp_file(&content, &args.url).await?;
605        let temp_path_str = temp_path.to_string_lossy().to_string();
606
607        // Preview prefers extracted readable text for HTML pages (scripts,
608        // styles, and head metadata removed); other bodies preview raw.
609        // The full body always lands in `temp_file` untouched.
610        let preview_limit = 8000;
611        let extracted_text = if html_preview::looks_like_html(&content) {
612            let text = html_preview::extract_text_preview(&content);
613            (!text.is_empty()).then_some(text)
614        } else {
615            None
616        };
617        let (preview, truncated, preview_kind, preview_basis_len) = match extracted_text {
618            Some(text) => {
619                let basis_len = text.len();
620                let truncated = basis_len > preview_limit;
621                let preview = if truncated {
622                    vtcode_commons::formatting::truncate_byte_budget(&text, preview_limit, "...")
623                } else {
624                    text
625                };
626                (preview, truncated, "text", basis_len)
627            }
628            None => {
629                let truncated = content_length > preview_limit;
630                let preview = if truncated {
631                    vtcode_commons::formatting::truncate_byte_budget(&content, preview_limit, "...")
632                } else {
633                    content.clone()
634                };
635                (preview, truncated, "raw", content_length)
636            }
637        };
638
639        // Cleanup: periodically remove old temp files (every ~50 calls)
640        if let Err(e) = cleanup_old_web_fetch_temps(TEMP_MAX_AGE_SECS).await {
641            tracing::debug!(error = %e, "Periodic web_fetch temp cleanup failed");
642        }
643
644        // Canonical response shape:
645        // - `temp_file`: path to ephemeral temp file containing full fetched body
646        // - `preview`: truncated snippet (8KB by default) for display
647        // - `prompt`: what the user/model wants to know
648        // - `next_action_hint`: explicit instruction so the agent continues the loop correctly
649        // - `no_spool: true` prevents the output spooler from persisting fetched content to disk
650        let mut response = json!({
651            "url": args.url,
652            "prompt": args.prompt,
653            "temp_file": temp_path_str,
654            "preview": preview,
655            "preview_kind": preview_kind,
656            "content_length": content_length,
657            "truncated": truncated,
658            "no_spool": true,
659            "next_action_hint": if preview_kind == "text" {
660                "Analyze the inline `preview` (readable text extracted from the page; scripts and styles removed) using `prompt` and answer the user directly. Only read `temp_file` (raw HTML) if you need content beyond the preview; it is ephemeral and may already be cleaned up."
661            } else {
662                "Analyze the inline `preview` (the start of the page) using `prompt` and answer the user directly. Only read `temp_file` if you need content beyond the preview; it is ephemeral and may already be cleaned up."
663            }
664        });
665
666        // Add overflow indicator if preview was truncated
667        if truncated {
668            response["overflow"] = json!(format!("[+{} more characters]", preview_basis_len - preview_limit));
669        }
670
671        if fetched.truncated_by_max_bytes {
672            response["truncated_by_max_bytes"] = json!(true);
673            response["max_bytes"] = json!(max_bytes);
674            response["source_size_bytes"] = json!(fetched.source_size_bytes);
675            response["next_action_hint"] = json!(
676                "Read `temp_file` to get the fetched content. Analyze it using `prompt`. If it does not contain enough detail, retry web_fetch with a larger max_bytes or a more specific URL."
677            );
678        }
679
680        Ok(response)
681    }
682}
683
684impl WebFetchTool {
685    /// Returns default headers used by the WebFetch client. This keeps Accept set to
686    /// prefer 'text/markdown' so documentation sites can provide token-efficient markdown
687    /// content as a preference.
688    fn default_headers() -> HeaderMap {
689        let mut headers = HeaderMap::new();
690        headers.insert(ACCEPT, HeaderValue::from_static("text/markdown, */*"));
691        headers.insert(USER_AGENT, HeaderValue::from_static("VT Code/1.0 (compatible; web-fetch tool)"));
692        headers
693    }
694}
695
696impl Default for WebFetchTool {
697    fn default() -> Self {
698        Self::new()
699    }
700}
701
702#[async_trait]
703impl Tool for WebFetchTool {
704    async fn execute(&self, mut args: Value) -> Result<Value> {
705        // Validate before dispatching to either backend. The markdown path is
706        // a convenience route through Defuddle, but it must have the same
707        // remote-only and SSRF policy as the normal fetch path.
708        if let Some(url) = args.get("url").and_then(Value::as_str) {
709            let max_bytes = args
710                .get("max_bytes")
711                .and_then(Value::as_u64)
712                .map(|value| value as usize)
713                .unwrap_or(MAX_CONTENT_SIZE);
714            let timeout_secs = args.get("timeout_secs").and_then(Value::as_u64).unwrap_or(DEFAULT_TIMEOUT_SECS);
715            if let Err(error) = self.validate_url(url) {
716                return Ok(web_fetch_error_response(url, max_bytes, timeout_secs, &error));
717            }
718        }
719
720        // Backwards-compatible argument normalization:
721        // - If called with only { "url": "..." } (no prompt), interpret as:
722        //   "Fetch this URL and return a concise natural language summary."
723        //
724        // This ensures:
725        // - Simple "fetch https://..." style calls are handled natively by VT Code.
726        // - We do not force upstream agents or MCP tools to construct a full prompt.
727        // - MCP tools like `get_current_time` remain unaffected (they are separate).
728        // `format: "markdown"` routes to the defuddle.md extraction backend
729        // (absorbs the former standalone `defuddle_fetch` tool). Session cap
730        // and URL validation are enforced by the backend itself.
731        if let Some(obj) = args.as_object_mut() {
732            let wants_markdown = obj
733                .get("format")
734                .and_then(Value::as_str)
735                .is_some_and(|f| f.eq_ignore_ascii_case("markdown"));
736            if wants_markdown {
737                let mut defuddle_args = serde_json::Map::new();
738                if let Some(url) = obj.get("url").cloned() {
739                    defuddle_args.insert("url".to_string(), url);
740                }
741                if let Some(max_bytes) = obj.get("max_bytes").cloned() {
742                    defuddle_args.insert("max_bytes".to_string(), max_bytes);
743                }
744                return self.defuddle.execute(Value::Object(defuddle_args)).await;
745            }
746        }
747
748        if let Some(obj) = args.as_object_mut() {
749            let has_url = obj.get("url").is_some_and(Value::is_string);
750            let has_prompt = obj.get("prompt").is_some_and(Value::is_string);
751
752            if has_url && !has_prompt {
753                obj.insert(
754                    "prompt".to_string(),
755                    json!("Summarize this page concisely. Read the temp_file to get the full content, then focus on the primary purpose, key information, and any actionable details."),
756                );
757            }
758        }
759
760        self.run(args).await
761    }
762
763    fn name(&self) -> &str {
764        tools::WEB_FETCH
765    }
766
767    fn description(&self) -> &str {
768        WEB_FETCH_DESCRIPTION
769    }
770}
771
772#[cfg(test)]
773mod tests;
vtcode_core/tools/web_fetch/mod.rs

vtcode_core/tools/web_fetch/
mod.rs
