hanzo_mcp/tools/
fetch_tool.rs

1/// Unified network tool (HIP-0300)
2///
3/// Handles network operations:
4/// - request: Full HTTP request with method/headers/body
5/// - fetch: Simplified GET returning text
6/// - head: HEAD request for headers only
7/// - download: Save URL to file
8/// - open: Open URL in browser
9/// - search: Web search query
10/// - crawl: Recursive site mirror
11
12use anyhow::{anyhow, Result};
13use serde::{Deserialize, Serialize};
14use serde_json::{json, Value};
15use std::collections::HashMap;
16
17use crate::hanzo_api::HanzoApi;
18
19#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
20#[serde(rename_all = "snake_case")]
21pub enum NetAction {
22    Request,
23    Fetch,
24    Head,
25    Download,
26    Open,
27    Search,
28    Crawl,
29    Help,
30}
31
32impl Default for NetAction {
33    fn default() -> Self {
34        Self::Help
35    }
36}
37
38impl std::str::FromStr for NetAction {
39    type Err = anyhow::Error;
40
41    fn from_str(s: &str) -> Result<Self> {
42        match s.to_lowercase().as_str() {
43            "request" | "get" => Ok(Self::Request),
44            "fetch" => Ok(Self::Fetch),
45            "head" => Ok(Self::Head),
46            "download" | "save" => Ok(Self::Download),
47            "open" | "browse" => Ok(Self::Open),
48            "search" => Ok(Self::Search),
49            "crawl" | "mirror" => Ok(Self::Crawl),
50            "help" | "" => Ok(Self::Help),
51            _ => Err(anyhow!("Unknown action: {}", s)),
52        }
53    }
54}
55
56#[derive(Debug, Clone, Default, Serialize, Deserialize)]
57pub struct FetchToolArgs {
58    pub action: Option<String>,
59    pub url: Option<String>,
60    pub method: Option<String>,
61    pub headers: Option<HashMap<String, String>>,
62    pub body: Option<String>,
63    pub output: Option<String>,
64    pub timeout: Option<u64>,
65    pub query: Option<String>,
66    pub depth: Option<usize>,
67    pub limit: Option<usize>,
68}
69
70pub struct FetchToolDefinition;
71
72impl FetchToolDefinition {
73    pub fn schema() -> Value {
74        json!({
75            "name": "fetch",
76            "description": "Network operations: request, fetch, head, download, open, search, crawl",
77            "inputSchema": {
78                "type": "object",
79                "properties": {
80                    "action": {
81                        "type": "string",
82                        "enum": ["request", "fetch", "head", "download", "open", "search", "crawl", "help"],
83                        "description": "Network action"
84                    },
85                    "url": { "type": "string", "description": "URL" },
86                    "method": { "type": "string", "description": "HTTP method", "default": "GET" },
87                    "headers": { "type": "object", "description": "HTTP headers" },
88                    "body": { "type": "string", "description": "Request body" },
89                    "output": { "type": "string", "description": "Output file/dir for download/crawl" },
90                    "timeout": { "type": "number", "description": "Timeout in ms", "default": 30000 },
91                    "query": { "type": "string", "description": "Search query" },
92                    "depth": { "type": "number", "description": "Crawl depth", "default": 2 },
93                    "limit": { "type": "number", "description": "Max results/pages", "default": 10 }
94                },
95                "required": ["action"]
96            }
97        })
98    }
99}
100
101pub struct FetchTool;
102
103impl FetchTool {
104    pub fn new() -> Self {
105        Self
106    }
107
108    fn build_client(&self, timeout: u64) -> Result<reqwest::Client> {
109        Ok(reqwest::Client::builder()
110            .timeout(std::time::Duration::from_millis(timeout))
111            .user_agent("Mozilla/5.0 (compatible; HanzoBot/1.0)")
112            .build()?)
113    }
114
115    pub async fn execute(&self, args: FetchToolArgs) -> Result<Value> {
116        let action: NetAction = args.action.as_deref().unwrap_or("help").parse()?;
117
118        match action {
119            NetAction::Request => self.request(&args).await,
120            NetAction::Fetch => self.fetch_url(&args).await,
121            NetAction::Head => self.head(&args).await,
122            NetAction::Download => self.download(&args).await,
123            NetAction::Open => self.open(&args).await,
124            NetAction::Search => self.search(&args).await,
125            NetAction::Crawl => self.crawl(&args).await,
126            NetAction::Help => Ok(self.help()),
127        }
128    }
129
130    async fn request(&self, args: &FetchToolArgs) -> Result<Value> {
131        let url = args.url.as_deref().ok_or_else(|| anyhow!("url required"))?;
132        let method = args.method.as_deref().unwrap_or("GET");
133        let timeout = args.timeout.unwrap_or(30000);
134        let client = self.build_client(timeout)?;
135
136        let mut req = match method.to_uppercase().as_str() {
137            "GET" => client.get(url),
138            "POST" => client.post(url),
139            "PUT" => client.put(url),
140            "DELETE" => client.delete(url),
141            "PATCH" => client.patch(url),
142            "HEAD" => client.head(url),
143            _ => return Err(anyhow!("Unsupported method: {}", method)),
144        };
145
146        if let Some(headers) = &args.headers {
147            for (k, v) in headers {
148                req = req.header(k.as_str(), v.as_str());
149            }
150        }
151        if let Some(body) = &args.body {
152            req = req.body(body.clone());
153        }
154
155        let resp = req.send().await?;
156        let status = resp.status().as_u16();
157        let headers: HashMap<String, String> = resp.headers()
158            .iter()
159            .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
160            .collect();
161
162        let content_type = headers.get("content-type").cloned().unwrap_or_default();
163        let body_text = resp.text().await?;
164        let body_val: Value = if content_type.contains("json") {
165            serde_json::from_str(&body_text).unwrap_or(Value::String(body_text.clone()))
166        } else {
167            Value::String(if body_text.len() > 50000 { body_text[..50000].to_string() } else { body_text })
168        };
169
170        Ok(json!({
171            "ok": true,
172            "data": { "status": status, "headers": headers, "body": body_val },
173            "error": null,
174            "meta": { "tool": "fetch", "action": "request" }
175        }))
176    }
177
178    async fn fetch_url(&self, args: &FetchToolArgs) -> Result<Value> {
179        let url = args.url.as_deref().ok_or_else(|| anyhow!("url required"))?;
180        let timeout = args.timeout.unwrap_or(30000);
181        let client = self.build_client(timeout)?;
182
183        let mut req = client.get(url);
184        if let Some(headers) = &args.headers {
185            for (k, v) in headers {
186                req = req.header(k.as_str(), v.as_str());
187            }
188        }
189
190        let resp = req.send().await?;
191        let status = resp.status().as_u16();
192        let headers: HashMap<String, String> = resp.headers()
193            .iter()
194            .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
195            .collect();
196        let text = resp.text().await?;
197        let text = if text.len() > 50000 { text[..50000].to_string() } else { text };
198
199        Ok(json!({
200            "ok": true,
201            "data": { "text": text, "status": status, "headers": headers },
202            "error": null,
203            "meta": { "tool": "fetch", "action": "fetch" }
204        }))
205    }
206
207    async fn head(&self, args: &FetchToolArgs) -> Result<Value> {
208        let url = args.url.as_deref().ok_or_else(|| anyhow!("url required"))?;
209        let timeout = args.timeout.unwrap_or(30000);
210        let client = self.build_client(timeout)?;
211
212        let mut req = client.head(url);
213        if let Some(headers) = &args.headers {
214            for (k, v) in headers {
215                req = req.header(k.as_str(), v.as_str());
216            }
217        }
218
219        let resp = req.send().await?;
220        let status = resp.status().as_u16();
221        let headers: HashMap<String, String> = resp.headers()
222            .iter()
223            .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
224            .collect();
225
226        Ok(json!({
227            "ok": true,
228            "data": { "status": status, "headers": headers },
229            "error": null,
230            "meta": { "tool": "fetch", "action": "head" }
231        }))
232    }
233
234    async fn download(&self, args: &FetchToolArgs) -> Result<Value> {
235        let url = args.url.as_deref().ok_or_else(|| anyhow!("url required"))?;
236        let output = args.output.as_deref().ok_or_else(|| anyhow!("output path required"))?;
237        let timeout = args.timeout.unwrap_or(60000);
238        let client = self.build_client(timeout)?;
239
240        let resp = client.get(url).send().await?;
241        if !resp.status().is_success() {
242            return Ok(json!({
243                "ok": false, "data": null,
244                "error": { "code": "HTTP_ERROR", "message": format!("{}", resp.status()) },
245                "meta": { "tool": "fetch", "action": "download" }
246            }));
247        }
248
249        let bytes = resp.bytes().await?;
250        if let Some(parent) = std::path::Path::new(output).parent() {
251            tokio::fs::create_dir_all(parent).await?;
252        }
253        tokio::fs::write(output, &bytes).await?;
254
255        Ok(json!({
256            "ok": true,
257            "data": { "url": url, "output": output, "size": bytes.len() },
258            "error": null,
259            "meta": { "tool": "fetch", "action": "download" }
260        }))
261    }
262
263    async fn open(&self, args: &FetchToolArgs) -> Result<Value> {
264        let url = args.url.as_deref().ok_or_else(|| anyhow!("url required"))?;
265
266        #[cfg(target_os = "macos")]
267        let cmd = "open";
268        #[cfg(target_os = "linux")]
269        let cmd = "xdg-open";
270        #[cfg(target_os = "windows")]
271        let cmd = "start";
272
273        tokio::process::Command::new(cmd).arg(url).output().await?;
274
275        Ok(json!({
276            "ok": true,
277            "data": { "url": url, "opened": true },
278            "error": null,
279            "meta": { "tool": "fetch", "action": "open" }
280        }))
281    }
282
283    async fn search(&self, args: &FetchToolArgs) -> Result<Value> {
284        let query = args.query.as_deref().ok_or_else(|| anyhow!("query required"))?;
285
286        // Canonical path: the platform web search (api.hanzo.ai). Fall back to a
287        // local DuckDuckGo scrape only when the cloud is unreachable / unkeyed.
288        let api = HanzoApi::from_env();
289        if api.has_key() {
290            if let Ok(body) = api.get("/v1/websearch/search", &[("q", query.to_string())]).await {
291                let limit = args.limit.unwrap_or(10);
292                let results: Vec<Value> = body["results"]
293                    .as_array()
294                    .map(|arr| {
295                        arr.iter()
296                            .take(limit)
297                            .map(|r| json!({
298                                "url": r.get("url").cloned().unwrap_or(Value::Null),
299                                "title": r.get("title").cloned().unwrap_or(Value::Null),
300                                "content": r.get("content").cloned().unwrap_or(Value::Null),
301                            }))
302                            .collect()
303                    })
304                    .unwrap_or_default();
305                return Ok(json!({
306                    "ok": true,
307                    "data": { "query": query, "results": results, "count": results.len(), "source": "cloud" },
308                    "error": null,
309                    "meta": { "tool": "fetch", "action": "search" }
310                }));
311            }
312        }
313
314        self.search_local(args).await
315    }
316
317    async fn search_local(&self, args: &FetchToolArgs) -> Result<Value> {
318        let query = args.query.as_deref().ok_or_else(|| anyhow!("query required"))?;
319        let timeout = args.timeout.unwrap_or(15000);
320        let client = self.build_client(timeout)?;
321
322        // Manual URL encoding for the query
323        let encoded_query: String = query.chars().map(|c| {
324            if c.is_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' {
325                c.to_string()
326            } else if c == ' ' {
327                "+".to_string()
328            } else {
329                format!("%{:02X}", c as u32)
330            }
331        }).collect();
332        let url = format!("https://html.duckduckgo.com/html/?q={}", encoded_query);
333        let resp = client.get(&url).send().await?;
334        let html = resp.text().await?;
335
336        // Simple regex extraction of search results
337        let re = regex::Regex::new(r#"class="result__title"[\s\S]*?href="([^"]*)"[^>]*>([\s\S]*?)</a>"#).unwrap();
338        let limit = args.limit.unwrap_or(10);
339        let mut results = Vec::new();
340
341        for cap in re.captures_iter(&html) {
342            if results.len() >= limit { break; }
343            let href = cap.get(1).map(|m| m.as_str()).unwrap_or("").replace("&amp;", "&");
344            let title = cap.get(2).map(|m| m.as_str()).unwrap_or("");
345            // Strip HTML tags from title
346            let tag_re = regex::Regex::new(r"<[^>]+>").unwrap();
347            let clean_title = tag_re.replace_all(title, "").trim().to_string();
348            results.push(json!({ "url": href, "title": clean_title }));
349        }
350
351        Ok(json!({
352            "ok": true,
353            "data": { "query": query, "results": results, "count": results.len(), "source": "local" },
354            "error": null,
355            "meta": { "tool": "fetch", "action": "search" }
356        }))
357    }
358
359    async fn crawl(&self, args: &FetchToolArgs) -> Result<Value> {
360        let url = args.url.as_deref().ok_or_else(|| anyhow!("url required"))?;
361        let output = args.output.as_deref().ok_or_else(|| anyhow!("output directory required"))?;
362        let max_depth = args.depth.unwrap_or(2);
363        let max_pages = args.limit.unwrap_or(100);
364        let timeout = args.timeout.unwrap_or(10000);
365        let client = self.build_client(timeout)?;
366
367        let start_host = reqwest::Url::parse(url)?.host_str().unwrap_or("").to_string();
368        let mut visited = std::collections::HashSet::new();
369        let mut pages = Vec::new();
370        let mut queue = vec![(url.to_string(), 0usize)];
371
372        tokio::fs::create_dir_all(output).await?;
373
374        while let Some((current_url, depth)) = queue.first().cloned() {
375            queue.remove(0);
376            if visited.contains(&current_url) || depth > max_depth || pages.len() >= max_pages {
377                continue;
378            }
379            if let Ok(parsed) = reqwest::Url::parse(&current_url) {
380                if parsed.host_str().unwrap_or("") != start_host { continue; }
381            } else { continue; }
382
383            visited.insert(current_url.clone());
384
385            let resp = match client.get(&current_url).send().await {
386                Ok(r) => r,
387                Err(_) => continue,
388            };
389            let body = match resp.text().await {
390                Ok(b) => b,
391                Err(_) => continue,
392            };
393
394            // Determine filename
395            if let Ok(parsed) = reqwest::Url::parse(&current_url) {
396                let path = parsed.path().trim_start_matches('/');
397                let file_name = if path.is_empty() || path.ends_with('/') {
398                    format!("{}index.html", path)
399                } else if !path.contains('.') {
400                    format!("{}.html", path)
401                } else {
402                    path.to_string()
403                };
404                let full_path = std::path::Path::new(output).join(&file_name);
405                if let Some(parent) = full_path.parent() {
406                    let _ = tokio::fs::create_dir_all(parent).await;
407                }
408                let _ = tokio::fs::write(&full_path, &body).await;
409                pages.push(full_path.display().to_string());
410            }
411
412            // Extract links for further crawling
413            let link_re = regex::Regex::new(r#"href=["']([^"']+)["']"#).unwrap();
414            for cap in link_re.captures_iter(&body) {
415                if let Some(href) = cap.get(1) {
416                    if let Ok(abs) = reqwest::Url::parse(&current_url).and_then(|base| base.join(href.as_str())) {
417                        let abs_str = abs.to_string();
418                        if !visited.contains(&abs_str) {
419                            queue.push((abs_str, depth + 1));
420                        }
421                    }
422                }
423            }
424        }
425
426        Ok(json!({
427            "ok": true,
428            "data": { "pages": pages, "count": pages.len(), "dest": output, "depth": max_depth },
429            "error": null,
430            "meta": { "tool": "fetch", "action": "crawl" }
431        }))
432    }
433
434    fn help(&self) -> Value {
435        json!({
436            "ok": true,
437            "data": {
438                "tool": "fetch",
439                "actions": {
440                    "request": "Full HTTP request (requires url, optional method/headers/body)",
441                    "fetch": "Simplified GET returning text (requires url)",
442                    "head": "HEAD request for headers only (requires url)",
443                    "download": "Save URL to file (requires url, output)",
444                    "open": "Open URL in browser (requires url)",
445                    "search": "Web search (requires query)",
446                    "crawl": "Recursive site mirror (requires url, output)"
447                }
448            },
449            "error": null,
450            "meta": { "tool": "fetch", "action": "help" }
451        })
452    }
453}
454
455#[cfg(test)]
456mod tests {
457    use super::*;
458
459    #[test]
460    fn test_net_action_parse() {
461        assert_eq!("request".parse::<NetAction>().unwrap(), NetAction::Request);
462        assert_eq!("fetch".parse::<NetAction>().unwrap(), NetAction::Fetch);
463        assert_eq!("head".parse::<NetAction>().unwrap(), NetAction::Head);
464        assert_eq!("download".parse::<NetAction>().unwrap(), NetAction::Download);
465        assert_eq!("search".parse::<NetAction>().unwrap(), NetAction::Search);
466        assert_eq!("crawl".parse::<NetAction>().unwrap(), NetAction::Crawl);
467    }
468}
hanzo_mcp/tools/fetch_tool.rs

hanzo_mcp/tools/
fetch_tool.rs
