syncable_cli/agent/tools/
fetch.rs

1//! Web fetch tool for retrieving online content
2//!
3//! Provides the agent with the ability to fetch content from URLs and convert
4//! HTML to readable markdown. Inspired by Forge's NetFetch tool.
5//!
6//! Features:
7//! - Fetches HTTP/HTTPS URLs
8//! - Converts HTML to markdown for readability
9//! - Respects robots.txt (basic check)
10//! - Truncates large responses to prevent context overflow
11//! - Returns raw content when requested
12
13use reqwest::{Client, Url};
14use rig::completion::ToolDefinition;
15use rig::tool::Tool;
16use serde::{Deserialize, Serialize};
17use serde_json::json;
18
19/// Maximum content length to return (characters)
20const MAX_CONTENT_LENGTH: usize = 40_000;
21
22// ============================================================================
23// Web Fetch Tool
24// ============================================================================
25
26#[derive(Debug, Deserialize)]
27pub struct WebFetchArgs {
28    /// URL to fetch
29    pub url: String,
30    /// If true, return raw content without markdown conversion (default: false)
31    pub raw: Option<bool>,
32}
33
34#[derive(Debug, thiserror::Error)]
35#[error("Web fetch error: {0}")]
36pub struct WebFetchError(String);
37
38#[derive(Debug, Clone, Serialize, Deserialize)]
39pub struct WebFetchTool {
40    #[serde(skip)]
41    client: Option<Client>,
42}
43
44impl Default for WebFetchTool {
45    fn default() -> Self {
46        Self::new()
47    }
48}
49
50impl WebFetchTool {
51    pub fn new() -> Self {
52        Self {
53            client: Some(
54                Client::builder()
55                    .user_agent("Mozilla/5.0 (compatible; SyncableCLI/0.1; +https://syncable.dev)")
56                    .timeout(std::time::Duration::from_secs(30))
57                    .build()
58                    .unwrap_or_default(),
59            ),
60        }
61    }
62
63    fn client(&self) -> Client {
64        self.client.clone().unwrap_or_default()
65    }
66
67    /// Check robots.txt for disallowed paths (basic check)
68    async fn check_robots_txt(&self, url: &Url) -> Result<(), WebFetchError> {
69        let robots_url = format!("{}://{}/robots.txt", url.scheme(), url.authority());
70
71        // Try to fetch robots.txt (ignore errors - many sites don't have one)
72        if let Ok(response) = self.client().get(&robots_url).send().await
73            && response.status().is_success()
74            && let Ok(robots_content) = response.text().await
75        {
76            let path = url.path();
77            for line in robots_content.lines() {
78                if let Some(disallowed) = line.strip_prefix("Disallow: ") {
79                    let disallowed = disallowed.trim();
80                    if !disallowed.is_empty() {
81                        let disallowed = if !disallowed.starts_with('/') {
82                            format!("/{}", disallowed)
83                        } else {
84                            disallowed.to_string()
85                        };
86                        let check_path = if !path.starts_with('/') {
87                            format!("/{}", path)
88                        } else {
89                            path.to_string()
90                        };
91                        if check_path.starts_with(&disallowed) {
92                            return Err(WebFetchError(format!(
93                                "URL {} cannot be fetched due to robots.txt restrictions",
94                                url
95                            )));
96                        }
97                    }
98                }
99            }
100        }
101        Ok(())
102    }
103
104    /// Fetch URL content and optionally convert HTML to markdown
105    async fn fetch_url(&self, url: &Url, force_raw: bool) -> Result<FetchResult, WebFetchError> {
106        // Check robots.txt first
107        self.check_robots_txt(url).await?;
108
109        let response = self
110            .client()
111            .get(url.as_str())
112            .send()
113            .await
114            .map_err(|e| WebFetchError(format!("Failed to fetch URL {}: {}", url, e)))?;
115
116        let status = response.status();
117        if !status.is_success() {
118            return Err(WebFetchError(format!(
119                "Failed to fetch {} - status code {}",
120                url, status
121            )));
122        }
123
124        let content_type = response
125            .headers()
126            .get("content-type")
127            .and_then(|v| v.to_str().ok())
128            .unwrap_or("")
129            .to_string();
130
131        let raw_content = response
132            .text()
133            .await
134            .map_err(|e| WebFetchError(format!("Failed to read response from {}: {}", url, e)))?;
135
136        // Determine if content is HTML
137        let is_html = raw_content[..100.min(raw_content.len())].contains("<html")
138            || raw_content[..100.min(raw_content.len())].contains("<!DOCTYPE")
139            || raw_content[..100.min(raw_content.len())].contains("<!doctype")
140            || content_type.contains("text/html")
141            || (content_type.is_empty() && raw_content.contains("<body"));
142
143        // Convert HTML to markdown unless raw is requested
144        let content = if is_html && !force_raw {
145            html_to_markdown(&raw_content)
146        } else {
147            raw_content
148        };
149
150        // Truncate if too long
151        let (content, was_truncated) = if content.len() > MAX_CONTENT_LENGTH {
152            (
153                content[..MAX_CONTENT_LENGTH].to_string() + "\n\n[Content truncated...]",
154                true,
155            )
156        } else {
157            (content, false)
158        };
159
160        Ok(FetchResult {
161            content,
162            content_type,
163            status_code: status.as_u16(),
164            was_truncated,
165            was_html: is_html && !force_raw,
166        })
167    }
168}
169
170#[derive(Debug)]
171struct FetchResult {
172    content: String,
173    content_type: String,
174    status_code: u16,
175    was_truncated: bool,
176    was_html: bool,
177}
178
179impl Tool for WebFetchTool {
180    const NAME: &'static str = "web_fetch";
181
182    type Error = WebFetchError;
183    type Args = WebFetchArgs;
184    type Output = String;
185
186    async fn definition(&self, _prompt: String) -> ToolDefinition {
187        ToolDefinition {
188            name: Self::NAME.to_string(),
189            description: r#"Fetch content from a URL and return it as text or markdown.
190
191Use this tool to:
192- Look up documentation for libraries, frameworks, or APIs
193- Check official guides and tutorials
194- Verify information from authoritative sources
195- Research best practices and patterns
196- Access API reference documentation
197- Get current information beyond training data
198
199The tool automatically converts HTML pages to readable markdown format.
200For API endpoints returning JSON/XML, use raw=true to get the unprocessed response.
201
202Limitations:
203- Cannot access pages requiring authentication
204- Respects robots.txt restrictions
205- Large pages are truncated to ~40,000 characters
206- Some sites may block automated requests"#
207                .to_string(),
208            parameters: json!({
209                "type": "object",
210                "properties": {
211                    "url": {
212                        "type": "string",
213                        "description": "The URL to fetch (must be http:// or https://)"
214                    },
215                    "raw": {
216                        "type": "boolean",
217                        "description": "If true, return raw content without HTML-to-markdown conversion. Default: false"
218                    }
219                },
220                "required": ["url"]
221            }),
222        }
223    }
224
225    async fn call(&self, args: Self::Args) -> Result<Self::Output, Self::Error> {
226        // Parse and validate URL
227        let url = Url::parse(&args.url)
228            .map_err(|e| WebFetchError(format!("Invalid URL '{}': {}", args.url, e)))?;
229
230        // Only allow http/https
231        if url.scheme() != "http" && url.scheme() != "https" {
232            return Err(WebFetchError(format!(
233                "Unsupported URL scheme '{}'. Only http and https are supported.",
234                url.scheme()
235            )));
236        }
237
238        let force_raw = args.raw.unwrap_or(false);
239        let result = self.fetch_url(&url, force_raw).await?;
240
241        let output = json!({
242            "url": args.url,
243            "status_code": result.status_code,
244            "content_type": result.content_type,
245            "converted_to_markdown": result.was_html,
246            "truncated": result.was_truncated,
247            "content": result.content
248        });
249
250        serde_json::to_string_pretty(&output)
251            .map_err(|e| WebFetchError(format!("Failed to serialize response: {}", e)))
252    }
253}
254
255/// Convert HTML content to Markdown
256///
257/// Uses a simple regex-based approach for common HTML elements.
258/// For more complex HTML, consider using a proper HTML parser.
259fn html_to_markdown(html: &str) -> String {
260    use regex::Regex;
261
262    let mut content = html.to_string();
263
264    // Remove script and style tags entirely
265    let script_re = Regex::new(r"(?is)<script[^>]*>.*?</script>").unwrap();
266    content = script_re.replace_all(&content, "").to_string();
267
268    let style_re = Regex::new(r"(?is)<style[^>]*>.*?</style>").unwrap();
269    content = style_re.replace_all(&content, "").to_string();
270
271    // Remove comments
272    let comment_re = Regex::new(r"(?is)<!--.*?-->").unwrap();
273    content = comment_re.replace_all(&content, "").to_string();
274
275    // Convert headers
276    let h1_re = Regex::new(r"(?is)<h1[^>]*>(.*?)</h1>").unwrap();
277    content = h1_re.replace_all(&content, "\n# $1\n").to_string();
278
279    let h2_re = Regex::new(r"(?is)<h2[^>]*>(.*?)</h2>").unwrap();
280    content = h2_re.replace_all(&content, "\n## $1\n").to_string();
281
282    let h3_re = Regex::new(r"(?is)<h3[^>]*>(.*?)</h3>").unwrap();
283    content = h3_re.replace_all(&content, "\n### $1\n").to_string();
284
285    let h4_re = Regex::new(r"(?is)<h4[^>]*>(.*?)</h4>").unwrap();
286    content = h4_re.replace_all(&content, "\n#### $1\n").to_string();
287
288    let h5_re = Regex::new(r"(?is)<h5[^>]*>(.*?)</h5>").unwrap();
289    content = h5_re.replace_all(&content, "\n##### $1\n").to_string();
290
291    let h6_re = Regex::new(r"(?is)<h6[^>]*>(.*?)</h6>").unwrap();
292    content = h6_re.replace_all(&content, "\n###### $1\n").to_string();
293
294    // Convert paragraphs
295    let p_re = Regex::new(r"(?is)<p[^>]*>(.*?)</p>").unwrap();
296    content = p_re.replace_all(&content, "\n$1\n").to_string();
297
298    // Convert links
299    let a_re = Regex::new(r#"(?is)<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>"#).unwrap();
300    content = a_re.replace_all(&content, "[$2]($1)").to_string();
301
302    // Convert bold/strong
303    let strong_re = Regex::new(r"(?is)<(?:strong|b)[^>]*>(.*?)</(?:strong|b)>").unwrap();
304    content = strong_re.replace_all(&content, "**$1**").to_string();
305
306    // Convert italic/em
307    let em_re = Regex::new(r"(?is)<(?:em|i)[^>]*>(.*?)</(?:em|i)>").unwrap();
308    content = em_re.replace_all(&content, "*$1*").to_string();
309
310    // Convert code blocks
311    let pre_re = Regex::new(r"(?is)<pre[^>]*><code[^>]*>(.*?)</code></pre>").unwrap();
312    content = pre_re.replace_all(&content, "\n```\n$1\n```\n").to_string();
313
314    let pre_only_re = Regex::new(r"(?is)<pre[^>]*>(.*?)</pre>").unwrap();
315    content = pre_only_re
316        .replace_all(&content, "\n```\n$1\n```\n")
317        .to_string();
318
319    // Convert inline code
320    let code_re = Regex::new(r"(?is)<code[^>]*>(.*?)</code>").unwrap();
321    content = code_re.replace_all(&content, "`$1`").to_string();
322
323    // Convert lists
324    let ul_re = Regex::new(r"(?is)<ul[^>]*>(.*?)</ul>").unwrap();
325    content = ul_re.replace_all(&content, "\n$1\n").to_string();
326
327    let ol_re = Regex::new(r"(?is)<ol[^>]*>(.*?)</ol>").unwrap();
328    content = ol_re.replace_all(&content, "\n$1\n").to_string();
329
330    let li_re = Regex::new(r"(?is)<li[^>]*>(.*?)</li>").unwrap();
331    content = li_re.replace_all(&content, "- $1\n").to_string();
332
333    // Convert blockquotes
334    let bq_re = Regex::new(r"(?is)<blockquote[^>]*>(.*?)</blockquote>").unwrap();
335    content = bq_re.replace_all(&content, "\n> $1\n").to_string();
336
337    // Convert line breaks
338    let br_re = Regex::new(r"(?i)<br\s*/?>").unwrap();
339    content = br_re.replace_all(&content, "\n").to_string();
340
341    // Convert horizontal rules
342    let hr_re = Regex::new(r"(?i)<hr\s*/?>").unwrap();
343    content = hr_re.replace_all(&content, "\n---\n").to_string();
344
345    // Remove remaining HTML tags
346    let tag_re = Regex::new(r"<[^>]+>").unwrap();
347    content = tag_re.replace_all(&content, "").to_string();
348
349    // Decode common HTML entities
350    content = content
351        .replace("&nbsp;", " ")
352        .replace("&lt;", "<")
353        .replace("&gt;", ">")
354        .replace("&amp;", "&")
355        .replace("&quot;", "\"")
356        .replace("&#39;", "'")
357        .replace("&apos;", "'")
358        .replace("&copy;", "©")
359        .replace("&reg;", "®")
360        .replace("&trade;", "™")
361        .replace("&mdash;", "—")
362        .replace("&ndash;", "–")
363        .replace("&hellip;", "…");
364
365    // Clean up excessive whitespace
366    let multiline_re = Regex::new(r"\n{3,}").unwrap();
367    content = multiline_re.replace_all(&content, "\n\n").to_string();
368
369    let space_re = Regex::new(r" {2,}").unwrap();
370    content = space_re.replace_all(&content, " ").to_string();
371
372    content.trim().to_string()
373}
374
375#[cfg(test)]
376mod tests {
377    use super::*;
378
379    #[test]
380    fn test_html_to_markdown_headers() {
381        let html = "<h1>Title</h1><h2>Subtitle</h2><h3>Section</h3>";
382        let md = html_to_markdown(html);
383        assert!(md.contains("# Title"));
384        assert!(md.contains("## Subtitle"));
385        assert!(md.contains("### Section"));
386    }
387
388    #[test]
389    fn test_html_to_markdown_links() {
390        let html = r#"<a href="https://example.com">Example</a>"#;
391        let md = html_to_markdown(html);
392        assert!(md.contains("[Example](https://example.com)"));
393    }
394
395    #[test]
396    fn test_html_to_markdown_formatting() {
397        let html = "<strong>bold</strong> and <em>italic</em>";
398        let md = html_to_markdown(html);
399        assert!(md.contains("**bold**"));
400        assert!(md.contains("*italic*"));
401    }
402
403    #[test]
404    fn test_html_to_markdown_code() {
405        let html = "<code>inline</code> and <pre><code>block</code></pre>";
406        let md = html_to_markdown(html);
407        assert!(md.contains("`inline`"));
408        assert!(md.contains("```"));
409    }
410
411    #[test]
412    fn test_html_to_markdown_lists() {
413        let html = "<ul><li>Item 1</li><li>Item 2</li></ul>";
414        let md = html_to_markdown(html);
415        assert!(md.contains("- Item 1"));
416        assert!(md.contains("- Item 2"));
417    }
418
419    #[test]
420    fn test_html_to_markdown_removes_scripts() {
421        let html = "<p>Content</p><script>alert('xss')</script><p>More</p>";
422        let md = html_to_markdown(html);
423        assert!(!md.contains("script"));
424        assert!(!md.contains("alert"));
425        assert!(md.contains("Content"));
426        assert!(md.contains("More"));
427    }
428}
syncable_cli/agent/tools/fetch.rs

syncable_cli/agent/tools/
fetch.rs
