microclaw_engine/tools/
web_fetch.rs

1use crate::internal::tool_runtime::web_content_validation::WebContentValidationConfig;
2use crate::internal::tool_runtime::web_fetch::WebFetchUrlValidationConfig;
3use async_trait::async_trait;
4use serde_json::json;
5
6use super::{schema_object, Tool, ToolResult};
7use microclaw_core::llm_types::ToolDefinition;
8
9pub struct WebFetchTool {
10    default_timeout_secs: u64,
11    validation: WebContentValidationConfig,
12    url_validation: WebFetchUrlValidationConfig,
13}
14
15impl WebFetchTool {
16    pub fn new(
17        default_timeout_secs: u64,
18        validation: WebContentValidationConfig,
19        url_validation: WebFetchUrlValidationConfig,
20    ) -> Self {
21        Self {
22            default_timeout_secs,
23            validation,
24            url_validation,
25        }
26    }
27}
28
29#[async_trait]
30impl Tool for WebFetchTool {
31    fn name(&self) -> &str {
32        "web_fetch"
33    }
34
35    fn definition(&self) -> ToolDefinition {
36        ToolDefinition {
37            name: "web_fetch".into(),
38            description:
39                "Fetch a URL and return its text content (HTML parsed, scripts/styles removed). Max 20KB."
40                    .into(),
41            input_schema: schema_object(
42                json!({
43                    "url": {
44                        "type": "string",
45                        "description": "The URL to fetch"
46                    },
47                    "timeout_secs": {
48                        "type": "integer",
49                        "description": "Timeout in seconds (defaults to configured tool timeout budget)"
50                    }
51                }),
52                &["url"],
53            ),
54        }
55    }
56
57    async fn execute(&self, input: serde_json::Value) -> ToolResult {
58        let url = match input.get("url").and_then(|v| v.as_str()) {
59            Some(u) => u,
60            None => return ToolResult::error("Missing required parameter: url".into()),
61        };
62        let timeout_secs = input
63            .get("timeout_secs")
64            .and_then(|v| v.as_u64())
65            .unwrap_or(self.default_timeout_secs);
66
67        match crate::internal::tool_runtime::web_fetch::fetch_url_with_timeout_and_validation(
68            url,
69            timeout_secs,
70            self.validation,
71            self.url_validation.clone(),
72        )
73        .await
74        {
75            Ok(text) => ToolResult::success(text),
76            Err(e) => ToolResult::error(format!("Failed to fetch URL: {e}")),
77        }
78    }
79}
80
81#[cfg(test)]
82mod tests {
83    use super::*;
84    use serde_json::json;
85
86    #[test]
87    fn test_web_fetch_definition() {
88        let tool = WebFetchTool::new(
89            15,
90            WebContentValidationConfig::default(),
91            WebFetchUrlValidationConfig::default(),
92        );
93        assert_eq!(tool.name(), "web_fetch");
94        let def = tool.definition();
95        assert_eq!(def.name, "web_fetch");
96        assert!(def.description.contains("20KB"));
97        assert!(def.input_schema["properties"]["url"].is_object());
98        let required = def.input_schema["required"].as_array().unwrap();
99        assert!(required.iter().any(|v| v == "url"));
100    }
101
102    #[tokio::test]
103    async fn test_web_fetch_missing_url() {
104        let tool = WebFetchTool::new(
105            15,
106            WebContentValidationConfig::default(),
107            WebFetchUrlValidationConfig::default(),
108        );
109        let result = tool.execute(json!({})).await;
110        assert!(result.is_error);
111        assert!(result.content.contains("Missing required parameter: url"));
112    }
113
114    #[tokio::test]
115    async fn test_web_fetch_null_url() {
116        let tool = WebFetchTool::new(
117            15,
118            WebContentValidationConfig::default(),
119            WebFetchUrlValidationConfig::default(),
120        );
121        let result = tool.execute(json!({"url": null})).await;
122        assert!(result.is_error);
123        assert!(result.content.contains("Missing required parameter: url"));
124    }
125
126    #[tokio::test]
127    async fn test_web_fetch_invalid_url() {
128        let tool = WebFetchTool::new(
129            1,
130            WebContentValidationConfig::default(),
131            WebFetchUrlValidationConfig::default(),
132        );
133        let result = tool
134            .execute(json!({"url": "https://this-domain-does-not-exist-12345.example"}))
135            .await;
136        assert!(result.is_error);
137        assert!(result.content.contains("Failed to fetch URL"));
138    }
139
140    #[tokio::test]
141    async fn test_web_fetch_blocks_disallowed_scheme_before_request() {
142        let tool = WebFetchTool::new(
143            15,
144            WebContentValidationConfig::default(),
145            WebFetchUrlValidationConfig::default(),
146        );
147        let result = tool.execute(json!({"url": "ftp://example.com"})).await;
148        assert!(result.is_error);
149        assert!(result.content.contains("not allowed"));
150    }
151
152    #[tokio::test]
153    async fn test_web_fetch_blocks_denylisted_host_before_request() {
154        let tool = WebFetchTool::new(
155            15,
156            WebContentValidationConfig::default(),
157            WebFetchUrlValidationConfig {
158                denylist_hosts: vec!["example.com".to_string()],
159                ..WebFetchUrlValidationConfig::default()
160            },
161        );
162        let result = tool.execute(json!({"url": "https://example.com"})).await;
163        assert!(result.is_error);
164        assert!(result.content.contains("denylisted"));
165    }
166}
microclaw_engine/tools/web_fetch.rs

microclaw_engine/tools/
web_fetch.rs
